From 7d66ddffd8c04e974a5e7819f1c569493502312e Mon Sep 17 00:00:00 2001 From: Laszlo Attila Toth Date: Wed, 9 Mar 2022 19:47:20 +0100 Subject: [PATCH 1/2] TEZ-4835: Update checkstyle configuration --- pom.xml | 10 ++-- .../main/resources/checkstyle/checkstyle.xml | 54 ++++++++++++++++--- 2 files changed, 52 insertions(+), 12 deletions(-) diff --git a/pom.xml b/pom.xml index c496e0ce87..00f9df3490 100644 --- a/pom.xml +++ b/pom.xml @@ -69,8 +69,8 @@ scm:git:https://gitbox.apache.org/repos/asf/tez.git 1.4 3.0.5 - 3.1.1 - 8.35 + 3.1.2 + 9.3 1.3.6 ${project.build.directory}/tmp @@ -907,10 +907,10 @@ - checkstyle/checkstyle.xml - checkstyle/suppressions.xml + tez-build-tools/src/main/resources/checkstyle/checkstyle.xml + tez-build-tools/src/main/resources/checkstyle/suppressions.xml true - false + true xml html ${project.build.directory}/test/checkstyle-errors.xml diff --git a/tez-build-tools/src/main/resources/checkstyle/checkstyle.xml b/tez-build-tools/src/main/resources/checkstyle/checkstyle.xml index cac5814e7f..c943263368 100644 --- a/tez-build-tools/src/main/resources/checkstyle/checkstyle.xml +++ b/tez-build-tools/src/main/resources/checkstyle/checkstyle.xml @@ -53,7 +53,8 @@ - + + @@ -66,7 +67,6 @@ - @@ -91,7 +91,10 @@ - + + + + @@ -119,6 +122,19 @@ + + + @@ -145,7 +161,13 @@ + + + + + + @@ -166,36 +188,54 @@ + + + + + + - + + + - + + + - - + + + + + + + + + + From 57ada410c095b4a82ee814b82947c06ce0ee44a5 Mon Sep 17 00:00:00 2001 From: Laszlo Attila Toth Date: Thu, 9 Jun 2022 17:07:46 +0200 Subject: [PATCH 2/2] TEZ-4835: partially reformat code to get closer to the expectations of checkstyle --- .../apache/tez/hadoop/shim/HadoopShim27.java | 1 - .../shim/TestHadoop25_26_27ShimProvider.java | 1 - .../apache/tez/hadoop/shim/HadoopShim28.java | 2 +- .../hadoop/shim/TestHadoopShim28Provider.java | 1 - .../apache/tez/hadoop/shim/HadoopShim.java | 4 +- .../tez/hadoop/shim/HadoopShimProvider.java | 8 +- .../tez/hadoop/shim/HadoopShimsLoader.java | 4 +- .../hadoop/shim/TestHadoopShimsLoader.java | 1 - .../apache/tez/client/AMConfiguration.java | 20 +- .../org/apache/tez/client/CallerContext.java | 33 +- .../apache/tez/client/FrameworkClient.java | 16 +- .../apache/tez/client/TezApiVersionInfo.java | 7 +- .../apache/tez/client/TezAppMasterStatus.java | 6 +- .../java/org/apache/tez/client/TezClient.java | 199 ++- .../org/apache/tez/client/TezClientUtils.java | 187 +- .../org/apache/tez/client/TezYarnClient.java | 10 +- .../org/apache/tez/common/ATSConstants.java | 8 +- .../org/apache/tez/common/CachedEntity.java | 6 +- .../tez/common/ContainerSignatureMatcher.java | 14 +- .../java/org/apache/tez/common/GuavaShim.java | 8 +- .../apache/tez/common/JavaOptsChecker.java | 6 +- .../org/apache/tez/common/Preconditions.java | 8 +- .../org/apache/tez/common/ProgressHelper.java | 5 +- .../java/org/apache/tez/common/RPCUtil.java | 38 +- .../apache/tez/common/ReflectionUtils.java | 10 +- .../tez/common/ServicePluginLifecycle.java | 1 - .../org/apache/tez/common/TezClassLoader.java | 4 +- .../org/apache/tez/common/TezCommonUtils.java | 50 +- .../java/org/apache/tez/common/TezUtils.java | 23 +- .../org/apache/tez/common/TezYARNUtils.java | 49 +- .../org/apache/tez/common/VersionInfo.java | 9 +- .../common/annotation/ConfigurationClass.java | 6 +- .../annotation/ConfigurationProperty.java | 8 +- .../tez/common/counters/AbstractCounter.java | 4 +- .../common/counters/AbstractCounterGroup.java | 6 +- .../tez/common/counters/AbstractCounters.java | 74 +- .../counters/AggregateFrameworkCounter.java | 17 +- .../common/counters/AggregateTezCounter.java | 11 +- .../counters/AggregateTezCounterDelegate.java | 10 +- .../common/counters/AggregateTezCounters.java | 17 +- .../tez/common/counters/CounterGroup.java | 6 +- .../tez/common/counters/CounterGroupBase.java | 25 +- .../common/counters/CounterGroupFactory.java | 28 +- .../tez/common/counters/DAGCounter.java | 8 +- .../counters/FileSystemCounterGroup.java | 27 +- .../counters/FrameworkCounterGroup.java | 7 +- .../tez/common/counters/GenericCounter.java | 5 +- .../tez/common/counters/JobCounter.java | 8 +- .../apache/tez/common/counters/Limits.java | 11 +- .../tez/common/counters/TaskCounter.java | 35 +- .../tez/common/counters/TezCounter.java | 14 +- .../tez/common/counters/TezCounters.java | 11 +- .../security/ACLConfigurationParser.java | 8 +- .../tez/common/security/ACLManager.java | 10 +- .../apache/tez/common/security/ACLType.java | 6 +- .../common/security/DAGAccessControls.java | 7 +- .../security/DAGClientSecurityInfo.java | 34 +- .../security/HistoryACLPolicyException.java | 15 +- .../security/HistoryACLPolicyManager.java | 12 +- .../common/security/JobTokenIdentifier.java | 12 +- .../security/JobTokenSecretManager.java | 13 +- .../apache/tez/common/security/Master.java | 8 +- .../tez/common/security/TokenCache.java | 47 +- .../tez/dag/api/ConfigurationScope.java | 6 +- .../main/java/org/apache/tez/dag/api/DAG.java | 79 +- .../tez/dag/api/DAGNotRunningException.java | 39 +- .../tez/dag/api/DAGSubmissionTimedOut.java | 7 +- .../apache/tez/dag/api/DagTypeConverters.java | 322 ++-- .../tez/dag/api/DataSinkDescriptor.java | 51 +- .../tez/dag/api/DataSourceDescriptor.java | 52 +- .../java/org/apache/tez/dag/api/Edge.java | 16 +- .../apache/tez/dag/api/EdgeManagerPlugin.java | 54 +- .../tez/dag/api/EdgeManagerPluginContext.java | 12 +- .../dag/api/EdgeManagerPluginDescriptor.java | 6 +- .../dag/api/EdgeManagerPluginOnDemand.java | 80 +- .../org/apache/tez/dag/api/EdgeProperty.java | 18 +- .../apache/tez/dag/api/EntityDescriptor.java | 12 +- .../apache/tez/dag/api/GroupInputEdge.java | 10 +- .../apache/tez/dag/api/HistoryLogLevel.java | 6 +- .../apache/tez/dag/api/InputDescriptor.java | 6 +- .../dag/api/InputInitializerDescriptor.java | 6 +- .../tez/dag/api/NamedEntityDescriptor.java | 2 +- .../dag/api/OutputCommitterDescriptor.java | 6 +- .../apache/tez/dag/api/OutputDescriptor.java | 8 +- .../org/apache/tez/dag/api/PreWarmVertex.java | 27 +- .../tez/dag/api/ProcessorDescriptor.java | 8 +- .../tez/dag/api/RootInputLeafOutput.java | 39 +- .../java/org/apache/tez/dag/api/Scope.java | 6 +- .../apache/tez/dag/api/SessionNotReady.java | 6 +- .../apache/tez/dag/api/SessionNotRunning.java | 7 +- .../apache/tez/dag/api/TaskLocationHint.java | 18 +- .../apache/tez/dag/api/TezConfiguration.java | 326 ++-- .../dag/api/TezConfigurationConstants.java | 7 +- .../org/apache/tez/dag/api/TezConstants.java | 27 +- .../org/apache/tez/dag/api/TezException.java | 39 +- .../tez/dag/api/TezReflectionException.java | 6 +- .../tez/dag/api/TezUncheckedException.java | 40 +- .../org/apache/tez/dag/api/UserPayload.java | 8 +- .../java/org/apache/tez/dag/api/Vertex.java | 73 +- .../org/apache/tez/dag/api/VertexGroup.java | 33 +- .../tez/dag/api/VertexLocationHint.java | 9 +- .../tez/dag/api/VertexManagerPlugin.java | 48 +- .../dag/api/VertexManagerPluginContext.java | 114 +- .../api/VertexManagerPluginDescriptor.java | 10 +- .../apache/tez/dag/api/client/DAGClient.java | 47 +- .../tez/dag/api/client/DAGClientImpl.java | 26 +- .../dag/api/client/DAGClientImplLocal.java | 10 +- .../tez/dag/api/client/DAGClientInternal.java | 46 +- .../dag/api/client/DAGClientTimelineImpl.java | 43 +- .../apache/tez/dag/api/client/DAGStatus.java | 109 +- .../apache/tez/dag/api/client/Progress.java | 49 +- .../tez/dag/api/client/StatusGetOpts.java | 6 +- .../dag/api/client/TimelineReaderFactory.java | 30 +- .../tez/dag/api/client/VertexStatus.java | 61 +- .../rpc/DAGClientAMProtocolBlockingPB.java | 10 +- .../dag/api/client/rpc/DAGClientRPCImpl.java | 48 +- .../tez/dag/api/client/rpc/package-info.java | 8 +- .../tez/dag/api/event/VertexStateUpdate.java | 3 +- .../VertexStateUpdateParallelismUpdated.java | 3 + .../api/AbstractLogicalIOProcessor.java | 4 +- .../tez/runtime/api/AbstractLogicalInput.java | 5 +- .../runtime/api/AbstractLogicalOutput.java | 5 +- .../apache/tez/runtime/api/DagIdentifier.java | 8 +- .../org/apache/tez/runtime/api/Event.java | 6 +- .../tez/runtime/api/ExecutionContext.java | 3 +- .../org/apache/tez/runtime/api/Input.java | 17 +- .../apache/tez/runtime/api/InputContext.java | 14 +- .../runtime/api/InputFrameworkInterface.java | 22 +- .../tez/runtime/api/InputInitializer.java | 10 +- .../runtime/api/InputInitializerContext.java | 18 +- .../tez/runtime/api/InputSpecUpdate.java | 20 +- .../tez/runtime/api/InputStatistics.java | 43 +- .../runtime/api/InputStatisticsReporter.java | 10 +- .../tez/runtime/api/LogicalIOProcessor.java | 6 +- .../LogicalIOProcessorFrameworkInterface.java | 11 +- .../apache/tez/runtime/api/LogicalInput.java | 8 +- .../api/LogicalInputFrameworkInterface.java | 2 +- .../apache/tez/runtime/api/LogicalOutput.java | 6 +- .../tez/runtime/api/MemoryUpdateCallback.java | 9 +- .../tez/runtime/api/MergedInputContext.java | 14 +- .../tez/runtime/api/MergedLogicalInput.java | 11 +- .../tez/runtime/api/ObjectRegistry.java | 23 +- .../org/apache/tez/runtime/api/Output.java | 21 +- .../tez/runtime/api/OutputCommitter.java | 15 +- .../runtime/api/OutputCommitterContext.java | 9 +- .../apache/tez/runtime/api/OutputContext.java | 9 +- .../runtime/api/OutputFrameworkInterface.java | 16 +- .../tez/runtime/api/OutputStatistics.java | 48 +- .../runtime/api/OutputStatisticsReporter.java | 9 +- .../org/apache/tez/runtime/api/Processor.java | 8 +- .../tez/runtime/api/ProcessorContext.java | 18 +- .../api/ProcessorFrameworkInterface.java | 7 +- .../runtime/api/ProgressFailedException.java | 7 +- .../org/apache/tez/runtime/api/Reader.java | 6 +- .../runtime/api/TaskAttemptIdentifier.java | 10 +- .../apache/tez/runtime/api/TaskContext.java | 29 +- .../tez/runtime/api/TaskIdentifier.java | 10 +- .../tez/runtime/api/VertexIdentifier.java | 10 +- .../tez/runtime/api/VertexStatistics.java | 37 +- .../org/apache/tez/runtime/api/Writer.java | 6 +- .../events/CompositeDataMovementEvent.java | 21 +- .../CompositeRoutedDataMovementEvent.java | 7 +- .../api/events/CustomProcessorEvent.java | 6 +- .../runtime/api/events/DataMovementEvent.java | 19 +- .../InputConfigureVertexTasksEvent.java | 7 +- .../api/events/InputDataInformationEvent.java | 22 +- .../runtime/api/events/InputFailedEvent.java | 18 +- .../api/events/InputInitializerEvent.java | 5 +- .../api/events/InputReadErrorEvent.java | 14 +- .../api/events/InputUpdatePayloadEvent.java | 6 +- .../api/events/VertexManagerEvent.java | 16 +- .../shuffle/api/ShuffleHandlerError.java | 32 +- .../common/shuffle/api/package-info.java | 8 +- .../api/ContainerLaunchRequest.java | 5 +- .../serviceplugins/api/ContainerLauncher.java | 4 +- .../api/ContainerLauncherContext.java | 3 - .../api/ContainerLauncherOperationBase.java | 4 +- .../tez/serviceplugins/api/DagInfo.java | 3 + .../api/ServicePluginContextBase.java | 5 +- .../api/ServicePluginError.java | 2 - .../api/ServicePluginErrorDefaults.java | 6 +- .../api/ServicePluginsDescriptor.java | 27 +- .../api/TaskCommunicatorDescriptor.java | 1 - .../tez/serviceplugins/api/TaskScheduler.java | 29 +- .../api/TaskSchedulerContext.java | 4 - .../org/apache/tez/client/TestTezClient.java | 197 +- .../apache/tez/client/TestTezClientUtils.java | 77 +- .../tez/common/TestJavaOptsChecker.java | 4 - .../org/apache/tez/common/TestRPCUtil.java | 16 +- .../tez/common/TestReflectionUtils.java | 14 +- .../apache/tez/common/TestTezCommonUtils.java | 38 +- .../apache/tez/common/TestTezYARNUtils.java | 12 +- .../apache/tez/common/TestVersionInfo.java | 7 +- .../security/TestACLConfigurationParser.java | 7 +- .../tez/common/security/TestACLManager.java | 40 +- .../security/TestDAGAccessControls.java | 13 +- .../tez/common/security/TestTokenCache.java | 14 +- .../java/org/apache/tez/dag/api/TestDAG.java | 28 +- .../org/apache/tez/dag/api/TestDAGPlan.java | 95 +- .../org/apache/tez/dag/api/TestDAGVerify.java | 57 +- .../tez/dag/api/TestDagTypeConverters.java | 14 +- .../tez/dag/api/TestEntityDescriptor.java | 18 +- .../tez/dag/api/TestHistoryLogLevel.java | 6 +- .../tez/dag/api/TestTaskLocationHint.java | 18 +- .../tez/dag/api/TestTezConfiguration.java | 8 +- .../tez/dag/api/client/TestATSHttpClient.java | 15 +- .../api/client/TestTimelineReaderFactory.java | 9 +- .../tez/dag/api/client/rpc/TestDAGClient.java | 202 ++- .../event/TestCompositeDataMovementEvent.java | 13 +- .../apache/tez/common/AsyncDispatcher.java | 69 +- .../tez/common/AsyncDispatcherConcurrent.java | 75 +- .../apache/tez/common/CallableWithNdc.java | 6 +- .../tez/common/DagContainerLauncher.java | 6 +- .../org/apache/tez/common/GcTimeUpdater.java | 6 +- .../org/apache/tez/common/Preconditions.java | 12 +- .../apache/tez/common/RunnableWithNdc.java | 6 +- .../org/apache/tez/common/StreamHelper.java | 6 +- .../apache/tez/common/TezAbstractEvent.java | 37 +- .../tez/common/TezContainerLogAppender.java | 10 +- .../org/apache/tez/common/TezExecutors.java | 2 +- .../tez/common/TezLog4jConfigurator.java | 6 +- .../apache/tez/common/TezSharedExecutor.java | 2 +- .../org/apache/tez/common/TezTestUtils.java | 8 +- .../apache/tez/common/TezUtilsInternal.java | 10 +- .../io/NonSyncByteArrayInputStream.java | 6 +- .../io/NonSyncByteArrayOutputStream.java | 12 +- .../common/io/NonSyncDataOutputStream.java | 6 +- .../org/apache/tez/common/package-info.java | 8 +- .../AbstractServletToControllerAdapter.java | 11 +- .../web/ServletToControllerAdapters.java | 6 +- .../apache/tez/common/web/package-info.java | 8 +- .../tez/dag/history/logging/EntityTypes.java | 6 +- .../apache/tez/dag/records/DAGIDAware.java | 14 +- .../tez/dag/records/DagIdentifierImpl.java | 19 +- .../tez/dag/records/TaskAttemptIDAware.java | 16 +- .../records/TaskAttemptIdentifierImpl.java | 21 +- .../records/TaskAttemptTerminationCause.java | 13 +- .../apache/tez/dag/records/TaskIDAware.java | 16 +- .../tez/dag/records/TaskIdentifierImpl.java | 19 +- .../org/apache/tez/dag/records/TezDAGID.java | 17 +- .../org/apache/tez/dag/records/TezID.java | 16 +- .../tez/dag/records/TezTaskAttemptID.java | 35 +- .../org/apache/tez/dag/records/TezTaskID.java | 29 +- .../apache/tez/dag/records/TezVertexID.java | 16 +- .../apache/tez/dag/records/VertexIDAware.java | 16 +- .../tez/dag/records/VertexIdentifierImpl.java | 21 +- .../apache/tez/dag/records/package-info.java | 8 +- .../tez/dag/utils/RelocalizationUtils.java | 10 +- .../apache/tez/dag/utils/package-info.java | 8 +- .../resources/InitialMemoryAllocator.java | 13 +- .../InitialMemoryRequestContext.java | 10 +- .../common/resources/package-info.java | 8 +- .../org/apache/tez/util/FastNumberFormat.java | 10 +- .../java/org/apache/tez/util/StopWatch.java | 7 +- .../tez/util/TezMxBeanResourceCalculator.java | 35 +- .../apache/tez/common/DrainDispatcher.java | 10 +- .../tez/common/MockDNSToSwitchMapping.java | 6 +- .../tez/common/TestAsyncDispatcher.java | 33 +- .../common/TestAsyncDispatcherConcurrent.java | 50 +- .../tez/common/TestTezSharedExecutor.java | 21 +- .../org/apache/tez/common/TestTezUtils.java | 48 +- .../apache/tez/dag/records/TestTezIds.java | 10 +- .../org/apache/tez/util/TestNumberFormat.java | 8 +- .../org/apache/tez/util/TestStopWatch.java | 7 +- .../util/TestTezMxBeanResourceCalculator.java | 9 +- .../src/main/java/org/apache/tez/Utils.java | 40 +- .../org/apache/tez/client/LocalClient.java | 93 +- .../tez/dag/api/client/DAGClientHandler.java | 44 +- .../tez/dag/api/client/DAGClientServer.java | 50 +- .../tez/dag/api/client/DAGStatusBuilder.java | 72 +- .../tez/dag/api/client/ProgressBuilder.java | 32 +- .../dag/api/client/VertexStatusBuilder.java | 36 +- ...GClientAMProtocolBlockingPBServerImpl.java | 74 +- .../apache/tez/dag/api/oldrecords/AMInfo.java | 16 +- .../dag/api/oldrecords/TaskAttemptReport.java | 37 +- .../dag/api/oldrecords/TaskAttemptState.java | 12 +- .../tez/dag/api/oldrecords/TaskReport.java | 15 +- .../tez/dag/api/oldrecords/TaskState.java | 6 +- .../org/apache/tez/dag/app/AppContext.java | 43 +- .../org/apache/tez/dag/app/ClusterInfo.java | 8 +- .../apache/tez/dag/app/ContainerContext.java | 31 +- .../dag/app/ContainerHeartbeatHandler.java | 39 +- .../dag/app/ContainerLauncherContextImpl.java | 3 - .../org/apache/tez/dag/app/DAGAppMaster.java | 541 +++--- .../apache/tez/dag/app/DAGAppMasterState.java | 32 +- .../tez/dag/app/HeartbeatHandlerBase.java | 74 +- .../apache/tez/dag/app/LocalDAGAppMaster.java | 12 +- .../apache/tez/dag/app/RecoveryParser.java | 104 +- ...ServicePluginLifecycleAbstractService.java | 1 + .../tez/dag/app/TaskAttemptEventInfo.java | 10 +- .../dag/app/TaskCommunicatorContextImpl.java | 2 +- .../tez/dag/app/TaskCommunicatorManager.java | 110 +- .../app/TaskCommunicatorManagerInterface.java | 37 +- .../tez/dag/app/TaskCommunicatorWrapper.java | 3 - .../tez/dag/app/TaskHeartbeatHandler.java | 36 +- .../apache/tez/dag/app/TezDagVersionInfo.java | 6 +- .../tez/dag/app/TezTaskCommunicatorImpl.java | 13 +- .../java/org/apache/tez/dag/app/dag/DAG.java | 54 +- .../org/apache/tez/dag/app/dag/DAGReport.java | 63 +- .../apache/tez/dag/app/dag/DAGScheduler.java | 53 +- .../org/apache/tez/dag/app/dag/DAGState.java | 34 +- .../tez/dag/app/dag/DAGTerminationCause.java | 43 +- .../app/dag/RootInputInitializerManager.java | 27 +- .../tez/dag/app/dag/StateChangeNotifier.java | 27 +- .../java/org/apache/tez/dag/app/dag/Task.java | 53 +- .../apache/tez/dag/app/dag/TaskAttempt.java | 58 +- .../dag/app/dag/TaskAttemptStateInternal.java | 40 +- .../tez/dag/app/dag/TaskStateInternal.java | 6 +- .../tez/dag/app/dag/TaskTerminationCause.java | 40 +- .../org/apache/tez/dag/app/dag/Vertex.java | 128 +- .../apache/tez/dag/app/dag/VertexState.java | 6 +- .../dag/app/dag/VertexTerminationCause.java | 40 +- .../tez/dag/app/dag/event/CallableEvent.java | 32 +- .../dag/app/dag/event/CallableEventType.java | 35 +- .../dag/app/dag/event/DAGAppMasterEvent.java | 34 +- .../event/DAGAppMasterEventDAGFinished.java | 34 +- ...GAppMasterEventSchedulingServiceError.java | 8 +- .../app/dag/event/DAGAppMasterEventType.java | 32 +- .../tez/dag/app/dag/event/DAGEvent.java | 33 +- .../dag/event/DAGEventCommitCompleted.java | 10 +- .../app/dag/event/DAGEventCounterUpdate.java | 16 +- .../dag/event/DAGEventDiagnosticsUpdate.java | 32 +- .../app/dag/event/DAGEventRecoverEvent.java | 12 +- .../dag/event/DAGEventSchedulerUpdate.java | 42 +- .../dag/app/dag/event/DAGEventStartDag.java | 36 +- .../tez/dag/app/dag/event/DAGEventType.java | 32 +- .../dag/event/DAGEventVertexCompleted.java | 8 +- .../dag/event/DAGEventVertexReRunning.java | 34 +- .../dag/app/dag/event/DiagnosableEvent.java | 6 +- .../tez/dag/app/dag/event/RecoveryEvent.java | 6 +- .../dag/app/dag/event/SpeculatorEvent.java | 37 +- ...peculatorEventTaskAttemptStatusUpdate.java | 47 +- .../app/dag/event/SpeculatorEventType.java | 33 +- .../dag/app/dag/event/TaskAttemptEvent.java | 36 +- .../event/TaskAttemptEventAttemptFailed.java | 38 +- .../event/TaskAttemptEventAttemptKilled.java | 14 +- .../TaskAttemptEventContainerTerminated.java | 6 +- ...temptEventContainerTerminatedBySystem.java | 38 +- .../TaskAttemptEventContainerTerminating.java | 7 +- .../event/TaskAttemptEventKillRequest.java | 39 +- .../dag/event/TaskAttemptEventNodeFailed.java | 11 +- .../event/TaskAttemptEventOutputFailed.java | 47 +- .../dag/event/TaskAttemptEventSchedule.java | 7 +- .../TaskAttemptEventStartedRemotely.java | 34 +- .../event/TaskAttemptEventStatusUpdate.java | 44 +- .../dag/event/TaskAttemptEventSubmitted.java | 1 - ...TaskAttemptEventTerminationCauseEvent.java | 32 +- .../event/TaskAttemptEventTezEventUpdate.java | 6 +- .../app/dag/event/TaskAttemptEventType.java | 37 +- .../tez/dag/app/dag/event/TaskEvent.java | 34 +- .../app/dag/event/TaskEventScheduleTask.java | 40 +- .../dag/app/dag/event/TaskEventTAFailed.java | 6 +- .../dag/app/dag/event/TaskEventTAKilled.java | 6 +- .../app/dag/event/TaskEventTALaunched.java | 6 +- .../app/dag/event/TaskEventTASucceeded.java | 6 +- .../dag/app/dag/event/TaskEventTAUpdate.java | 32 +- .../app/dag/event/TaskEventTermination.java | 35 +- .../tez/dag/app/dag/event/TaskEventType.java | 35 +- .../tez/dag/app/dag/event/VertexEvent.java | 32 +- .../dag/event/VertexEventCommitCompleted.java | 8 +- .../VertexEventInputDataInformation.java | 38 +- .../VertexEventManagerUserCodeError.java | 37 +- .../event/VertexEventNullEdgeInitialized.java | 38 +- .../dag/event/VertexEventRecoverVertex.java | 30 +- .../dag/event/VertexEventRootInputFailed.java | 38 +- .../VertexEventRootInputInitialized.java | 37 +- .../app/dag/event/VertexEventRouteEvent.java | 35 +- ...VertexEventSourceTaskAttemptCompleted.java | 34 +- .../event/VertexEventSourceVertexStarted.java | 42 +- .../VertexEventTaskAttemptCompleted.java | 38 +- .../dag/event/VertexEventTaskCompleted.java | 33 +- .../dag/event/VertexEventTaskReschedule.java | 33 +- .../app/dag/event/VertexEventTermination.java | 35 +- .../dag/app/dag/event/VertexEventType.java | 49 +- .../dag/event/VertexShuffleDataDeletion.java | 9 +- .../tez/dag/app/dag/event/package-info.java | 1 + .../dag/app/dag/impl/AMUserCodeException.java | 13 +- .../app/dag/impl/BroadcastEdgeManager.java | 61 +- .../apache/tez/dag/app/dag/impl/DAGImpl.java | 648 ++++--- .../dag/impl/DAGSchedulerNaturalOrder.java | 51 +- .../DAGSchedulerNaturalOrderControlled.java | 1 - .../org/apache/tez/dag/app/dag/impl/Edge.java | 342 ++-- .../dag/impl/ImmediateStartVertexManager.java | 39 +- .../dag/app/dag/impl/OneToOneEdgeManager.java | 48 +- .../dag/impl/OneToOneEdgeManagerOnDemand.java | 59 +- .../dag/impl/OutputCommitterContextImpl.java | 19 +- .../app/dag/impl/RootInputVertexManager.java | 81 +- .../dag/impl/ScatterGatherEdgeManager.java | 61 +- .../dag/app/dag/impl/ServicePluginInfo.java | 1 - .../tez/dag/app/dag/impl/TaskAttemptImpl.java | 235 ++- .../app/dag/impl/TaskAttemptImplHelpers.java | 12 +- .../apache/tez/dag/app/dag/impl/TaskImpl.java | 439 +++-- .../tez/dag/app/dag/impl/TaskReportImpl.java | 9 +- .../TezRootInputInitializerContextImpl.java | 13 +- .../tez/dag/app/dag/impl/VertexImpl.java | 1384 +++++++-------- .../tez/dag/app/dag/impl/VertexManager.java | 152 +- .../VertexShuffleDataDeletionContext.java | 6 +- .../tez/dag/app/dag/impl/VertexStats.java | 27 +- .../tez/dag/app/dag/impl/package-info.java | 1 + .../apache/tez/dag/app/dag/package-info.java | 1 + .../speculation/legacy/DataStatistics.java | 38 +- .../speculation/legacy/LegacySpeculator.java | 103 +- .../legacy/LegacyTaskRuntimeEstimator.java | 39 +- ...SimpleExponentialTaskRuntimeEstimator.java | 17 +- .../speculation/legacy/StartEndTimesBase.java | 36 +- .../legacy/TaskRuntimeEstimator.java | 36 +- .../forecast/SimpleExponentialSmoothing.java | 35 +- .../legacy/forecast/package-info.java | 1 + .../launcher/ContainerLauncherManager.java | 14 +- .../launcher/ContainerLauncherWrapper.java | 2 +- .../dag/app/launcher/DagDeleteRunnable.java | 6 +- .../tez/dag/app/launcher/DeletionTracker.java | 6 +- .../dag/app/launcher/DeletionTrackerImpl.java | 11 +- .../app/launcher/LocalContainerLauncher.java | 27 +- .../TaskAttemptFailedDeleteRunnable.java | 10 +- .../launcher/TezContainerLauncherImpl.java | 83 +- .../app/launcher/VertexDeleteRunnable.java | 10 +- .../tez/dag/app/launcher/package-info.java | 1 + .../org/apache/tez/dag/app/package-info.java | 1 + .../tez/dag/app/rm/AMSchedulerEvent.java | 5 +- .../AMSchedulerEventDeallocateContainer.java | 9 +- .../AMSchedulerEventNodeBlacklistUpdate.java | 6 +- .../dag/app/rm/AMSchedulerEventTAEnded.java | 6 +- .../rm/AMSchedulerEventTALaunchRequest.java | 16 +- .../tez/dag/app/rm/AMSchedulerEventType.java | 8 +- .../tez/dag/app/rm/ContainerAllocator.java | 35 +- .../dag/app/rm/ContainerLauncherEvent.java | 6 +- .../app/rm/ContainerLauncherEventType.java | 6 +- .../ContainerLauncherLaunchRequestEvent.java | 8 +- .../rm/ContainerLauncherStopRequestEvent.java | 7 +- .../dag/app/rm/DagAwareYarnTaskScheduler.java | 279 +-- .../dag/app/rm/LocalTaskSchedulerService.java | 76 +- .../dag/app/rm/TaskSchedulerContextImpl.java | 2 - .../rm/TaskSchedulerContextImplWrapper.java | 37 +- .../tez/dag/app/rm/TaskSchedulerManager.java | 132 +- .../tez/dag/app/rm/TezAMRMClientAsync.java | 45 +- .../dag/app/rm/YarnTaskSchedulerService.java | 562 +++--- .../app/rm/YarnTaskSchedulerServiceError.java | 1 - .../tez/dag/app/rm/container/AMContainer.java | 45 +- .../app/rm/container/AMContainerEvent.java | 8 +- .../container/AMContainerEventAssignTA.java | 12 +- .../container/AMContainerEventCompleted.java | 48 +- .../AMContainerEventLaunchFailed.java | 11 +- .../AMContainerEventLaunchRequest.java | 10 +- .../container/AMContainerEventLaunched.java | 5 +- .../container/AMContainerEventNodeFailed.java | 7 +- .../container/AMContainerEventStopFailed.java | 6 +- .../AMContainerEventStopRequest.java | 5 +- .../AMContainerEventTASucceeded.java | 8 +- .../rm/container/AMContainerEventType.java | 44 +- .../app/rm/container/AMContainerHelpers.java | 46 +- .../dag/app/rm/container/AMContainerImpl.java | 419 +++-- .../dag/app/rm/container/AMContainerMap.java | 11 +- .../app/rm/container/AMContainerState.java | 6 +- .../dag/app/rm/container/AMContainerTask.java | 10 +- .../rm/container/ContainerContextMatcher.java | 40 +- .../apache/tez/dag/app/rm/node/AMNode.java | 12 +- .../tez/dag/app/rm/node/AMNodeEvent.java | 6 +- .../node/AMNodeEventContainerAllocated.java | 5 +- .../node/AMNodeEventContainerCompleted.java | 4 +- .../rm/node/AMNodeEventNodeCountUpdated.java | 10 +- .../app/rm/node/AMNodeEventStateChanged.java | 11 +- .../rm/node/AMNodeEventTaskAttemptEnded.java | 14 +- .../node/AMNodeEventTaskAttemptSucceeded.java | 7 +- .../tez/dag/app/rm/node/AMNodeEventType.java | 37 +- .../tez/dag/app/rm/node/AMNodeImpl.java | 243 ++- .../tez/dag/app/rm/node/AMNodeState.java | 6 +- .../tez/dag/app/rm/node/AMNodeTracker.java | 49 +- .../dag/app/rm/node/PerSourceNodeTracker.java | 2 - .../apache/tez/dag/app/rm/package-info.java | 1 + .../authorize/TezAMPolicyProvider.java | 27 +- .../app/security/authorize/package-info.java | 1 + .../tez/dag/app/web/AMWebController.java | 88 +- .../apache/tez/dag/app/web/WebUIService.java | 8 +- .../tez/dag/history/DAGHistoryEvent.java | 9 +- .../apache/tez/dag/history/HistoryEvent.java | 7 +- .../tez/dag/history/HistoryEventHandler.java | 23 +- .../tez/dag/history/HistoryEventType.java | 7 +- .../tez/dag/history/RecoveryConverters.java | 7 +- .../apache/tez/dag/history/SummaryEvent.java | 7 +- .../dag/history/events/AMLaunchedEvent.java | 9 +- .../dag/history/events/AMStartedEvent.java | 9 +- .../dag/history/events/AppLaunchedEvent.java | 11 +- .../events/ContainerLaunchedEvent.java | 11 +- .../history/events/ContainerStoppedEvent.java | 21 +- .../history/events/DAGCommitStartedEvent.java | 7 +- .../dag/history/events/DAGFinishedEvent.java | 16 +- .../history/events/DAGInitializedEvent.java | 9 +- .../history/events/DAGKillRequestEvent.java | 12 +- .../dag/history/events/DAGRecoveredEvent.java | 19 +- .../dag/history/events/DAGStartedEvent.java | 8 +- .../dag/history/events/DAGSubmittedEvent.java | 17 +- .../events/TaskAttemptFinishedEvent.java | 54 +- .../events/TaskAttemptStartedEvent.java | 17 +- .../dag/history/events/TaskFinishedEvent.java | 12 +- .../dag/history/events/TaskStartedEvent.java | 10 +- .../events/VertexCommitStartedEvent.java | 8 +- .../events/VertexConfigurationDoneEvent.java | 28 +- .../history/events/VertexFinishedEvent.java | 14 +- .../VertexGroupCommitFinishedEvent.java | 11 +- .../events/VertexGroupCommitStartedEvent.java | 10 +- .../events/VertexInitializedEvent.java | 23 +- .../history/events/VertexStartedEvent.java | 8 +- .../logging/HistoryLoggingService.java | 7 +- .../impl/DevNullHistoryLoggingService.java | 1 - .../impl/HistoryEventJsonConversion.java | 27 +- .../impl/SimpleHistoryLoggingService.java | 7 +- .../dag/history/recovery/RecoveryService.java | 37 +- .../tez/dag/history/utils/DAGUtils.java | 118 +- .../tez/dag/history/utils/TezEventUtils.java | 19 +- .../java/org/apache/tez/dag/utils/Graph.java | 12 +- .../org/apache/tez/dag/utils/ProtoUtils.java | 15 +- .../utils/Simple2LevelVersionComparator.java | 7 +- .../utils/TaskSpecificLaunchCmdOption.java | 14 +- .../apache/tez/dag/utils/TezBuilderUtils.java | 15 +- .../tez/dag/utils/TezRuntimeChildJVM.java | 44 +- .../serviceplugins/api/TaskCommunicator.java | 27 +- .../api/TaskCommunicatorContext.java | 5 +- .../api/TaskHeartbeatRequest.java | 1 - .../tez/state/OnStateChangedCallback.java | 3 +- .../dag/api/client/TestDAGClientHandler.java | 27 +- .../dag/api/client/TestDAGClientServer.java | 6 +- .../api/client/TestVertexStatusBuilder.java | 7 +- ...GClientAMProtocolBlockingPBServerImpl.java | 8 +- .../org/apache/tez/dag/app/MockClock.java | 36 +- .../apache/tez/dag/app/MockDAGAppMaster.java | 126 +- .../apache/tez/dag/app/MockLocalClient.java | 44 +- .../org/apache/tez/dag/app/MockTezClient.java | 59 +- .../tez/dag/app/PluginWrapperTestHelpers.java | 3 - .../apache/tez/dag/app/TestDAGAppMaster.java | 29 +- .../tez/dag/app/TestMemoryWithEvents.java | 67 +- .../tez/dag/app/TestMockDAGAppMaster.java | 299 ++-- .../apache/tez/dag/app/TestPreemption.java | 100 +- .../tez/dag/app/TestRecoveryParser.java | 124 +- .../apache/tez/dag/app/TestSpeculation.java | 74 +- .../dag/app/TestTaskCommunicatorManager.java | 15 +- .../dag/app/TestTaskCommunicatorManager1.java | 98 +- .../dag/app/TestTaskCommunicatorManager2.java | 4 - .../dag/app/TestTaskCommunicatorWrapper.java | 1 - .../dag/TestRootInputInitializerManager.java | 6 +- .../dag/app/dag/TestStateChangeNotifier.java | 16 +- .../app/TestTezTaskCommunicatorManager.java | 5 +- .../app/dag/impl/CallableEventDispatcher.java | 24 +- .../tez/dag/app/dag/impl/TestCommit.java | 87 +- .../tez/dag/app/dag/impl/TestDAGImpl.java | 995 ++++++----- .../tez/dag/app/dag/impl/TestDAGRecovery.java | 307 ++-- .../dag/app/dag/impl/TestDAGScheduler.java | 51 +- ...estDAGSchedulerNaturalOrderControlled.java | 7 - .../apache/tez/dag/app/dag/impl/TestEdge.java | 70 +- .../impl/TestImmediateStartVertexManager.java | 45 +- .../dag/impl/TestRootInputVertexManager.java | 32 +- .../tez/dag/app/dag/impl/TestTaskAttempt.java | 105 +- .../tez/dag/app/dag/impl/TestTaskImpl.java | 122 +- .../tez/dag/app/dag/impl/TestVertexImpl.java | 1577 ++++++++--------- .../tez/dag/app/dag/impl/TestVertexImpl2.java | 14 +- .../dag/app/dag/impl/TestVertexManager.java | 59 +- .../tez/dag/app/dag/impl/TestVertexStats.java | 1 - .../legacy/TestDataStatistics.java | 40 +- .../TestContainerLauncherManager.java | 8 +- .../TestContainerLauncherWrapper.java | 1 - .../dag/app/launcher/TestDeletionTracker.java | 6 +- .../launcher/TestTezLocalCacheManager.java | 2 - .../tez/dag/app/rm/TestContainerReuse.java | 233 ++- .../app/rm/TestDagAwareYarnTaskScheduler.java | 63 +- .../dag/app/rm/TestLocalTaskScheduler.java | 17 +- .../app/rm/TestLocalTaskSchedulerService.java | 46 +- .../tez/dag/app/rm/TestTaskScheduler.java | 286 ++- .../dag/app/rm/TestTaskSchedulerHelpers.java | 30 +- .../dag/app/rm/TestTaskSchedulerManager.java | 54 +- .../tez/dag/app/rm/TestTezAMRMClient.java | 33 +- .../dag/app/rm/container/TestAMContainer.java | 105 +- .../app/rm/container/TestAMContainerMap.java | 13 +- .../dag/app/rm/node/TestAMNodeTracker.java | 69 +- .../tez/dag/app/web/TestAMWebController.java | 69 +- .../dag/history/TestHistoryEventHandler.java | 9 +- .../tez/dag/history/TestHistoryEventType.java | 1 - .../TestHistoryEventsProtoConversion.java | 65 +- .../impl/TestHistoryEventJsonConversion.java | 13 +- .../history/recovery/TestRecoveryService.java | 50 +- .../tez/dag/history/utils/TestDAGUtils.java | 13 +- .../TestSimple2LevelVersionComparator.java | 8 +- .../TestTaskSpecificLaunchCmdOption.java | 10 +- .../ControlledScheduledExecutorService.java | 16 +- .../apache/tez/test/EdgeManagerForTest.java | 17 +- .../GraceShuffleVertexManagerForTest.java | 2 +- .../tez/test/VertexManagerPluginForTest.java | 24 +- .../apache/tez/examples/CartesianProduct.java | 57 +- .../apache/tez/examples/ExampleDriver.java | 12 +- .../apache/tez/examples/HashJoinExample.java | 14 +- .../org/apache/tez/examples/JoinDataGen.java | 18 +- .../org/apache/tez/examples/JoinValidate.java | 15 +- .../apache/tez/examples/OrderedWordCount.java | 31 +- .../tez/examples/SimpleSessionExample.java | 10 +- .../tez/examples/SortMergeJoinExample.java | 20 +- .../apache/tez/examples/TezExampleBase.java | 15 +- .../org/apache/tez/examples/WordCount.java | 25 +- .../dag/app/TezTestServiceCommunicator.java | 8 +- .../TezTestServiceContainerLauncher.java | 3 - .../TezTestServiceNoOpContainerLauncher.java | 1 - .../TezTestServiceTaskSchedulerService.java | 4 - .../TezTestServiceTaskCommunicatorImpl.java | 4 +- .../tez/examples/JoinValidateConfigured.java | 1 - .../apache/tez/service/ContainerRunner.java | 1 + .../service/MiniTezTestServiceCluster.java | 7 +- .../service/TezTestServiceConfConstants.java | 26 +- .../tez/service/impl/ContainerRunnerImpl.java | 24 +- .../tez/service/impl/TezTestService.java | 3 - .../TezTestServiceProtocolClientImpl.java | 5 +- .../TezTestServiceProtocolServerImpl.java | 2 - .../apache/tez/shufflehandler/IndexCache.java | 27 +- .../tez/shufflehandler/ShuffleHandler.java | 88 +- .../tests/ExternalTezServiceTestHelper.java | 2 - .../tests/TestExtServicesWithLocalMode.java | 12 +- .../tez/tests/TestExternalTezServices.java | 8 +- .../tests/TestExternalTezServicesErrors.java | 10 +- .../org/apache/tez/util/ProtoConverters.java | 3 - .../mapred/split/SplitSizeEstimator.java | 8 +- .../hadoop/mapred/split/TezGroupedSplit.java | 43 +- .../split/TezGroupedSplitsInputFormat.java | 57 +- .../mapred/split/TezMapredSplitsGrouper.java | 16 +- .../split/SplitMetaInfoReaderTez.java | 12 +- .../mapreduce/split/SplitSizeEstimator.java | 6 +- .../mapreduce/split/TezGroupedSplit.java | 50 +- .../split/TezGroupedSplitsInputFormat.java | 57 +- .../split/TezMapReduceSplitsGrouper.java | 17 +- .../tez/dag/api/client/MRDAGClient.java | 3 +- .../tez/mapreduce/client/ClientCache.java | 9 +- .../client/ClientServiceDelegate.java | 24 +- .../tez/mapreduce/client/DAGJobStatus.java | 73 +- .../tez/mapreduce/client/NotRunningJob.java | 57 +- .../mapreduce/client/ResourceMgrDelegate.java | 56 +- .../tez/mapreduce/client/YARNRunner.java | 130 +- .../client/YarnTezClientProtocolProvider.java | 7 +- .../tez/mapreduce/client/package-info.java | 8 +- .../tez/mapreduce/combine/MRCombiner.java | 64 +- .../committer/MROutputCommitter.java | 18 +- .../common/MRInputAMSplitGenerator.java | 19 +- .../common/MRInputSplitDistributor.java | 10 +- .../apache/tez/mapreduce/common/Utils.java | 7 +- .../grouper/GroupedSplitContainer.java | 3 - .../grouper/MapReduceSplitContainer.java | 1 - .../grouper/MapredSplitContainer.java | 1 - .../tez/mapreduce/grouper/SplitContainer.java | 1 - .../SplitLocationProviderWrapperMapred.java | 2 +- .../grouper/SplitSizeEstimatorWrapper.java | 2 - .../SplitSizeEstimatorWrapperMapReduce.java | 4 +- .../SplitSizeEstimatorWrapperMapred.java | 2 +- .../mapreduce/grouper/TezSplitGrouper.java | 59 +- .../tez/mapreduce/hadoop/DeprecatedKeys.java | 98 +- .../tez/mapreduce/hadoop/IDConverter.java | 45 +- .../tez/mapreduce/hadoop/InputSplitInfo.java | 17 +- .../mapreduce/hadoop/InputSplitInfoDisk.java | 13 +- .../mapreduce/hadoop/InputSplitInfoMem.java | 24 +- .../apache/tez/mapreduce/hadoop/MRConfig.java | 64 +- .../tez/mapreduce/hadoop/MRHelpers.java | 28 +- .../tez/mapreduce/hadoop/MRInputHelpers.java | 28 +- .../tez/mapreduce/hadoop/MRJobConfig.java | 167 +- .../MultiStageMRConfToTezTranslator.java | 17 +- .../hadoop/MultiStageMRConfigUtil.java | 18 +- .../mapreduce/hadoop/TezTypeConverters.java | 10 +- .../hadoop/mapred/JobContextImpl.java | 16 +- .../mapreduce/hadoop/mapred/MRCounters.java | 82 +- .../mapreduce/hadoop/mapred/MRReporter.java | 14 +- .../hadoop/mapred/TaskAttemptContextImpl.java | 18 +- .../mapreduce/hadoop/mapred/package-info.java | 8 +- .../hadoop/mapreduce/JobContextImpl.java | 123 +- .../hadoop/mapreduce/MapContextImpl.java | 18 +- .../mapreduce/TaskAttemptContextImpl.java | 36 +- .../mapreduce/TaskInputOutputContextImpl.java | 30 +- .../mapreduce/TezNullOutputCommitter.java | 6 +- .../hadoop/mapreduce/package-info.java | 8 +- .../tez/mapreduce/hadoop/package-info.java | 8 +- .../apache/tez/mapreduce/input/MRInput.java | 94 +- .../tez/mapreduce/input/MRInputLegacy.java | 21 +- .../tez/mapreduce/input/MultiMRInput.java | 12 +- .../tez/mapreduce/input/base/MRInputBase.java | 9 +- .../tez/mapreduce/lib/MRInputUtils.java | 12 +- .../apache/tez/mapreduce/lib/MRReader.java | 21 +- .../tez/mapreduce/lib/MRReaderMapReduce.java | 12 +- .../tez/mapreduce/lib/MRReaderMapred.java | 14 +- .../tez/mapreduce/lib/package-info.java | 8 +- .../apache/tez/mapreduce/output/MROutput.java | 58 +- .../tez/mapreduce/output/MROutputLegacy.java | 6 +- .../tez/mapreduce/output/MultiMROutput.java | 32 +- .../mapreduce/partition/MRPartitioner.java | 8 +- .../tez/mapreduce/processor/MRTask.java | 111 +- .../mapreduce/processor/MRTaskReporter.java | 43 +- .../processor/SimpleMRProcessor.java | 4 +- .../mapreduce/processor/map/MapProcessor.java | 81 +- .../processor/reduce/ReduceProcessor.java | 97 +- .../mapred/split/TestGroupedSplits.java | 307 ++-- .../apache/tez/mapreduce/TestUmbilical.java | 7 +- .../apache/tez/mapreduce/TezTestUtils.java | 18 +- .../tez/mapreduce/combine/TestMRCombiner.java | 18 +- .../common/TestMRInputAMSplitGenerator.java | 25 +- .../common/TestMRInputSplitDistributor.java | 7 +- .../hadoop/TestConfigTranslationMRToTez.java | 12 +- .../mapreduce/hadoop/TestDeprecatedKeys.java | 7 +- .../tez/mapreduce/hadoop/TestMRHelpers.java | 9 +- .../mapreduce/hadoop/TestMRInputHelpers.java | 4 - .../tez/mapreduce/input/MRInputForTest.java | 6 +- .../mapreduce/input/MultiMRInputForTest.java | 6 +- .../tez/mapreduce/input/TestMRInput.java | 3 - .../tez/mapreduce/input/TestMultiMRInput.java | 17 +- .../mapreduce/lib/TestKVReadersWithMR.java | 2 - .../tez/mapreduce/output/TestMROutput.java | 43 +- .../output/TestMROutputConfigBuilder.java | 21 +- .../mapreduce/output/TestMROutputLegacy.java | 14 +- .../mapreduce/output/TestMultiMROutput.java | 17 +- .../tez/mapreduce/processor/MapUtils.java | 71 +- .../processor/map/TestMapProcessor.java | 39 +- .../processor/reduce/TestReduceProcessor.java | 32 +- .../tez/auxservices/FadvisedChunkedFile.java | 6 +- .../tez/auxservices/FadvisedFileRegion.java | 18 +- .../apache/tez/auxservices/IndexCache.java | 26 +- .../tez/auxservices/ShuffleHandler.java | 261 +-- .../tez/auxservices/TestIndexCache.java | 49 +- .../tez/auxservices/TestShuffleHandler.java | 235 +-- .../auxservices/TestShuffleHandlerJobs.java | 18 +- .../org/apache/tez/history/ATSImportTool.java | 10 +- .../apache/tez/history/parser/ATSData.java | 7 +- .../tez/history/parser/ATSFileParser.java | 8 +- .../history/parser/SimpleHistoryParser.java | 172 +- .../AdditionalInputOutputDetails.java | 8 +- .../history/parser/datamodel/BaseInfo.java | 8 +- .../history/parser/datamodel/BaseParser.java | 7 +- .../history/parser/datamodel/Constants.java | 9 +- .../history/parser/datamodel/Container.java | 15 +- .../tez/history/parser/datamodel/DagInfo.java | 37 +- .../history/parser/datamodel/EdgeInfo.java | 10 +- .../tez/history/parser/datamodel/Event.java | 6 +- .../parser/datamodel/TaskAttemptInfo.java | 63 +- .../history/parser/datamodel/TaskInfo.java | 35 +- .../history/parser/datamodel/VersionInfo.java | 1 - .../history/parser/datamodel/VertexInfo.java | 71 +- .../tez/history/parser/utils/Utils.java | 18 +- .../apache/tez/history/TestHistoryParser.java | 30 +- .../logging/proto/DatePartitionedLogger.java | 6 +- .../proto/HistoryEventProtoConverter.java | 100 +- .../HistoryEventProtoJsonConversion.java | 132 +- .../proto/ProtoHistoryLoggingService.java | 2 +- .../logging/proto/ProtoMessageWritable.java | 2 + .../proto/TestDagManifestFileScanner.java | 5 +- .../proto/TestHistoryEventProtoConverter.java | 36 +- .../proto/TestProtoHistoryLoggingService.java | 10 +- .../logging/ats/TimelineCachePluginImpl.java | 10 +- .../ats/TestTimelineCachePluginImpl.java | 4 +- .../ats/acls/ATSHistoryACLPolicyManager.java | 26 +- .../ats/acls/TestATSHistoryWithACLs.java | 56 +- .../acls/ATSV15HistoryACLPolicyManager.java | 26 +- .../ats/ATSV15HistoryLoggingService.java | 20 +- .../history/ats/acls/TestATSHistoryV15.java | 13 +- .../ats/TestATSV15HistoryLoggingService.java | 89 +- .../logging/ats/ATSHistoryLoggingService.java | 37 +- .../ats/HistoryEventTimelineConversion.java | 24 +- .../ats/TestATSHistoryLoggingService.java | 71 +- .../ats/TestATSHistoryWithMiniCluster.java | 9 +- .../TestHistoryEventTimelineConversion.java | 74 +- .../tez/tests/MiniTezClusterWithTimeline.java | 54 +- .../apache/tez/common/ContainerContext.java | 6 +- .../org/apache/tez/common/ContainerTask.java | 17 +- .../apache/tez/common/ProtoConverters.java | 22 +- .../apache/tez/common/TezConverterUtils.java | 7 +- .../apache/tez/common/TezLocalResource.java | 6 +- .../tez/common/TezTaskUmbilicalProtocol.java | 7 +- .../apache/tez/runtime/InputReadyTracker.java | 11 +- .../LogicalIOProcessorRuntimeTask.java | 142 +- .../org/apache/tez/runtime/RuntimeTask.java | 28 +- .../api/events/TaskAttemptCompletedEvent.java | 7 +- .../api/events/TaskAttemptFailedEvent.java | 7 +- .../api/events/TaskAttemptKilledEvent.java | 7 +- .../api/events/TaskStatusUpdateEvent.java | 15 +- .../tez/runtime/api/impl/EventMetaData.java | 17 +- .../tez/runtime/api/impl/EventType.java | 6 +- .../tez/runtime/api/impl/GroupInputSpec.java | 27 +- .../tez/runtime/api/impl/IOStatistics.java | 15 +- .../tez/runtime/api/impl/InputSpec.java | 9 +- .../tez/runtime/api/impl/OutputSpec.java | 8 +- .../apache/tez/runtime/api/impl/TaskSpec.java | 27 +- .../tez/runtime/api/impl/TaskStatistics.java | 10 +- .../runtime/api/impl/TezCountersDelegate.java | 10 +- .../apache/tez/runtime/api/impl/TezEvent.java | 253 ++- .../runtime/api/impl/TezHeartbeatRequest.java | 11 +- .../api/impl/TezHeartbeatResponse.java | 16 +- .../runtime/api/impl/TezInputContextImpl.java | 19 +- .../api/impl/TezMergedInputContextImpl.java | 12 +- .../api/impl/TezOutputContextImpl.java | 36 +- .../api/impl/TezProcessorContextImpl.java | 21 +- .../runtime/api/impl/TezTaskContextImpl.java | 28 +- .../tez/runtime/api/impl/TezUmbilical.java | 7 +- .../objectregistry/ObjectRegistryImpl.java | 13 +- .../common/resources/MemoryDistributor.java | 57 +- .../common/resources/ScalingAllocator.java | 18 +- .../common/security/JobTokenSelector.java | 8 +- .../internals/api/TaskReporterInterface.java | 9 +- .../runtime/internals/api/TezTrapEvent.java | 8 +- .../internals/api/TezTrapEventType.java | 6 +- .../metrics/FileSystemStatisticUpdater.java | 4 +- .../runtime/metrics/TaskCounterUpdater.java | 32 +- .../tez/runtime/task/ContainerReporter.java | 12 +- .../tez/runtime/task/ErrorReporter.java | 8 +- .../apache/tez/runtime/task/TaskReporter.java | 30 +- .../tez/runtime/task/TaskRunner2Callable.java | 8 +- .../org/apache/tez/runtime/task/TezChild.java | 47 +- .../tez/runtime/task/TezTaskRunner2.java | 32 +- .../tez/runtime/task/TezTrapEventHandler.java | 36 +- .../tez/runtime/TestInputReadyTracker.java | 44 +- .../TestLogicalIOProcessorRuntimeTask.java | 45 +- .../api/impl/TestProcessorContext.java | 12 +- .../tez/runtime/api/impl/TestTaskSpec.java | 5 +- .../tez/runtime/api/impl/TestTezEvent.java | 8 +- .../objectregistry/TestObjectRegistry.java | 11 +- .../resources/TestMemoryDistributor.java | 71 +- .../task/TaskExecutionTestHelpers.java | 1 - .../runtime/task/TestContainerExecution.java | 1 - .../tez/runtime/task/TestTaskExecution2.java | 13 +- .../tez/runtime/task/TestTaskReporter.java | 12 +- .../tez/runtime/task/TestTezTaskRunner2.java | 5 +- .../java/org/apache/hadoop/io/FileChunk.java | 8 +- .../edgemanager/SilentEdgeManager.java | 6 +- .../DestinationTaskInputsProperty.java | 42 +- .../vertexmanager/FairEdgeConfiguration.java | 39 +- .../vertexmanager/FairShuffleEdgeManager.java | 41 +- .../FairShuffleVertexManager.java | 74 +- .../InputReadyVertexManager.java | 80 +- .../vertexmanager/ShuffleVertexManager.java | 183 +- .../ShuffleVertexManagerBase.java | 140 +- .../VertexManagerWithConcurrentInput.java | 10 +- .../apache/tez/http/BaseHttpConnection.java | 1 - .../org/apache/tez/http/HttpConnection.java | 6 +- .../apache/tez/http/HttpConnectionParams.java | 3 +- .../java/org/apache/tez/http/SSLFactory.java | 5 +- .../http/async/netty/AsyncHttpConnection.java | 6 +- .../netty/TezBodyDeferringAsyncHandler.java | 4 +- .../runtime/library/api/KeyValueReader.java | 14 +- .../runtime/library/api/KeyValueWriter.java | 8 +- .../api/KeyValueWriterWithBasePath.java | 6 +- .../runtime/library/api/KeyValuesReader.java | 13 +- .../runtime/library/api/KeyValuesWriter.java | 6 +- .../tez/runtime/library/api/Partitioner.java | 19 +- .../library/api/TezRuntimeConfiguration.java | 29 +- .../CartesianProductCombination.java | 28 +- .../CartesianProductConfig.java | 80 +- .../CartesianProductEdgeManager.java | 20 +- ...artesianProductEdgeManagerPartitioned.java | 26 +- .../CartesianProductEdgeManagerReal.java | 14 +- .../CartesianProductFilter.java | 6 +- .../CartesianProductFilterDescriptor.java | 8 +- .../CartesianProductVertexManager.java | 66 +- ...tesianProductVertexManagerPartitioned.java | 34 +- .../CartesianProductVertexManagerReal.java | 6 +- .../FairCartesianProductEdgeManager.java | 25 +- .../FairCartesianProductVertexManager.java | 61 +- .../CompositeInputAttemptIdentifier.java | 9 +- .../runtime/library/common/ConfigUtils.java | 40 +- .../tez/runtime/library/common/Constants.java | 17 +- .../common/InputAttemptIdentifier.java | 14 +- .../library/common/InputIdentifier.java | 8 +- .../common/MemoryUpdateCallbackHandler.java | 8 +- .../library/common/TezRuntimeUtils.java | 84 +- .../library/common/ValuesIterator.java | 86 +- .../library/common/combine/Combiner.java | 10 +- .../common/comparator/ProxyComparator.java | 7 +- .../common/comparator/TezBytesComparator.java | 25 +- .../common/readers/UnorderedKVReader.java | 39 +- .../common/security/SecureShuffleUtils.java | 17 +- .../serializer/SerializationContext.java | 12 +- .../TezBytesWritableSerialization.java | 8 +- .../common/shuffle/DiskFetchedInput.java | 18 +- .../library/common/shuffle/FetchResult.java | 10 +- .../library/common/shuffle/FetchedInput.java | 20 +- .../common/shuffle/FetchedInputAllocator.java | 13 +- .../common/shuffle/FetchedInputCallback.java | 13 +- .../library/common/shuffle/Fetcher.java | 141 +- .../common/shuffle/FetcherCallback.java | 13 +- .../shuffle/FetcherErrorTestingConfig.java | 6 +- .../shuffle/FetcherWithInjectableErrors.java | 10 +- .../library/common/shuffle/HostPort.java | 6 +- .../shuffle/InputAttemptFetchFailure.java | 8 +- .../library/common/shuffle/InputHost.java | 12 +- .../common/shuffle/LocalDiskFetchedInput.java | 11 +- .../common/shuffle/MemoryFetchedInput.java | 14 +- .../common/shuffle/ShuffleEventHandler.java | 7 +- .../library/common/shuffle/ShuffleUtils.java | 61 +- .../impl/ShuffleInputEventHandlerImpl.java | 69 +- .../common/shuffle/impl/ShuffleManager.java | 83 +- .../impl/SimpleFetchedInputAllocator.java | 73 +- .../common/shuffle/impl/package-info.java | 8 +- .../orderedgrouped/ExceptionReporter.java | 7 +- .../FetchedInputAllocatorOrderedGrouped.java | 6 +- .../orderedgrouped/FetcherOrderedGrouped.java | 69 +- ...herOrderedGroupedWithInjectableErrors.java | 14 +- .../orderedgrouped/InMemoryReader.java | 27 +- .../orderedgrouped/InMemoryWriter.java | 9 +- .../shuffle/orderedgrouped/MapHost.java | 18 +- .../shuffle/orderedgrouped/MapOutput.java | 44 +- .../shuffle/orderedgrouped/MergeManager.java | 320 ++-- .../shuffle/orderedgrouped/MergeThread.java | 26 +- .../shuffle/orderedgrouped/Shuffle.java | 77 +- .../shuffle/orderedgrouped/ShuffleHeader.java | 23 +- ...huffleInputEventHandlerOrderedGrouped.java | 15 +- .../orderedgrouped/ShuffleScheduler.java | 93 +- .../shuffle/orderedgrouped/package-info.java | 8 +- .../library/common/shuffle/package-info.java | 8 +- .../common/sort/impl/ExternalSorter.java | 57 +- .../library/common/sort/impl/IFile.java | 85 +- .../common/sort/impl/IFileInputStream.java | 65 +- .../common/sort/impl/IFileOutputStream.java | 18 +- .../common/sort/impl/PipelinedSorter.java | 318 ++-- .../common/sort/impl/TezIndexRecord.java | 34 +- .../library/common/sort/impl/TezMerger.java | 431 +++-- .../sort/impl/TezRawKeyValueIterator.java | 28 +- .../common/sort/impl/TezSpillRecord.java | 16 +- .../common/sort/impl/dflt/DefaultSorter.java | 204 +-- .../common/sort/impl/dflt/package-info.java | 8 +- .../common/sort/impl/package-info.java | 8 +- .../task/local/output/TezTaskOutput.java | 12 +- .../task/local/output/TezTaskOutputFiles.java | 21 +- .../task/local/output/package-info.java | 8 +- .../BaseUnorderedPartitionedKVWriter.java | 47 +- .../writers/UnorderedPartitionedKVWriter.java | 65 +- .../library/common/writers/package-info.java | 8 +- .../HadoopKeyValuesBasedBaseEdgeConfig.java | 11 +- .../conf/OrderedGroupedKVInputConfig.java | 14 +- .../conf/OrderedPartitionedKVEdgeConfig.java | 10 +- .../OrderedPartitionedKVOutputConfig.java | 23 +- .../library/conf/UnorderedKVEdgeConfig.java | 10 +- .../library/conf/UnorderedKVInputConfig.java | 8 +- .../library/conf/UnorderedKVOutputConfig.java | 5 +- .../UnorderedPartitionedKVEdgeConfig.java | 6 +- .../UnorderedPartitionedKVOutputConfig.java | 6 +- .../tez/runtime/library/conf/Utils.java | 10 +- .../runtime/library/conf/package-info.java | 8 +- .../FetcherReadTimeoutException.java | 8 +- .../InputAlreadyClosedException.java | 8 +- .../hadoop/compat/NullProgressable.java | 33 +- .../ConcatenatedMergedKeyValueInput.java | 10 +- .../ConcatenatedMergedKeyValuesInput.java | 14 +- .../input/OrderedGroupedInputLegacy.java | 74 +- .../library/input/OrderedGroupedKVInput.java | 26 +- .../input/OrderedGroupedMergedKVInput.java | 21 +- .../library/input/UnorderedKVInput.java | 37 +- .../runtime/library/input/package-info.java | 8 +- .../output/OrderedPartitionedKVOutput.java | 11 +- .../library/output/UnorderedKVOutput.java | 39 +- .../output/UnorderedPartitionedKVOutput.java | 8 +- .../runtime/library/output/package-info.java | 8 +- .../library/partitioner/HashPartitioner.java | 7 +- .../partitioner/RoundRobinPartitioner.java | 18 +- .../library/processor/PreWarmProcessor.java | 12 +- .../library/processor/SimpleProcessor.java | 15 +- .../library/processor/SleepProcessor.java | 19 +- .../WeightedScalingMemoryDistributor.java | 26 +- .../runtime/library/utils/BufferUtils.java | 7 +- .../tez/runtime/library/utils/CodecUtils.java | 74 +- .../library/utils/DATA_RANGE_IN_MB.java | 1 - .../library/utils/FastByteComparisons.java | 17 +- .../tez/runtime/library/utils/Grouper.java | 16 +- .../runtime/library/utils/LocalProgress.java | 6 +- .../TestFairShuffleVertexManager.java | 47 +- .../TestInputReadyVertexManager.java | 65 +- .../TestShuffleVertexManager.java | 26 +- .../TestShuffleVertexManagerBase.java | 43 +- .../TestShuffleVertexManagerUtils.java | 35 +- .../TestVertexManagerWithConcurrentInput.java | 6 +- .../apache/tez/http/TestHttpConnection.java | 4 +- .../TestWeightedScalingMemoryDistributor.java | 13 +- .../api/TestTezRuntimeConfiguration.java | 3 - .../TestCartesianProductCombination.java | 45 +- .../TestCartesianProductConfig.java | 28 +- .../TestCartesianProductEdgeManager.java | 22 +- ...artesianProductEdgeManagerPartitioned.java | 32 +- .../TestCartesianProductVertexManager.java | 37 +- ...tesianProductVertexManagerPartitioned.java | 29 +- .../TestFairCartesianProductEdgeManager.java | 116 +- ...TestFairCartesianProductVertexManager.java | 70 +- .../library/cartesianproduct/TestGrouper.java | 6 +- .../library/common/TestConfigUtils.java | 9 +- .../library/common/TestInputIdentifiers.java | 11 +- .../library/common/TestValuesIterator.java | 53 +- .../comparator/TestProxyComparator.java | 22 +- .../common/readers/TestUnorderedKVReader.java | 6 +- .../library/common/shuffle/TestFetcher.java | 24 +- .../common/shuffle/TestShuffleUtils.java | 33 +- .../TestShuffleInputEventHandlerImpl.java | 22 +- .../shuffle/impl/TestShuffleManager.java | 37 +- .../impl/TestSimpleFetchedInputAllocator.java | 28 +- .../shuffle/orderedgrouped/TestFetcher.java | 123 +- .../orderedgrouped/TestMergeManager.java | 70 +- .../shuffle/orderedgrouped/TestShuffle.java | 5 - ...huffleInputEventHandlerOrderedGrouped.java | 26 +- .../orderedgrouped/TestShuffleScheduler.java | 31 +- .../library/common/sort/impl/TestIFile.java | 69 +- .../common/sort/impl/TestPipelinedSorter.java | 81 +- .../common/sort/impl/TestTezMerger.java | 99 +- .../sort/impl/dflt/TestDefaultSorter.java | 43 +- .../TestUnorderedPartitionedKVWriter.java | 112 +- ...TestOrderedGroupedMergedKVInputConfig.java | 1 - .../TestOrderedPartitionedKVEdgeConfig.java | 22 +- .../conf/TestUnorderedKVEdgeConfig.java | 14 +- .../TestUnorderedPartitionedKVEdgeConfig.java | 12 +- ...estUnorderedPartitionedKVOutputConfig.java | 8 +- .../input/TestOrderedGroupedKVInput.java | 5 - .../input/TestSortedGroupedMergedInput.java | 113 +- .../library/output/OutputTestHelpers.java | 5 +- .../output/TestOnFileSortedOutput.java | 91 +- .../output/TestOnFileUnorderedKVOutput.java | 23 +- .../TestOrderedPartitionedKVOutput2.java | 2 +- .../output/TestUnorderedKVOutput2.java | 4 +- .../TestUnorderedPartitionedKVOutput2.java | 1 - .../runtime/library/testutils/KVDataGen.java | 8 +- .../library/testutils/RuntimeTestUtils.java | 6 +- .../runtime/library/utils/TestCodecUtils.java | 6 +- .../examples/BroadcastAndOneToOneExample.java | 50 +- .../mapreduce/examples/BroadcastLoadGen.java | 10 +- .../mapreduce/examples/CartesianProduct.java | 42 +- .../tez/mapreduce/examples/ExampleDriver.java | 41 +- .../mapreduce/examples/FilterLinesByWord.java | 25 +- .../examples/FilterLinesByWordOneToOne.java | 20 +- .../apache/tez/mapreduce/examples/Join.java | 70 +- .../tez/mapreduce/examples/MRRSleepJob.java | 215 ++- .../mapreduce/examples/MapredWordCount.java | 17 +- .../examples/MultipleCommitsExample.java | 64 +- .../tez/mapreduce/examples/RPCLoadGen.java | 4 +- .../mapreduce/examples/RandomTextWriter.java | 1128 ++++++------ .../tez/mapreduce/examples/RandomWriter.java | 124 +- .../tez/mapreduce/examples/SecondarySort.java | 65 +- .../apache/tez/mapreduce/examples/Sort.java | 76 +- .../examples/TestOrderedWordCount.java | 95 +- .../tez/mapreduce/examples/UnionExample.java | 70 +- .../helpers/SplitsInClientOptionParser.java | 7 +- .../processor/FilterByWordInputProcessor.java | 18 +- .../FilterByWordOutputProcessor.java | 14 +- .../org/apache/tez/mapreduce/TestMRRJobs.java | 72 +- .../tez/mapreduce/TestMRRJobsDAGApi.java | 61 +- .../apache/tez/test/AMShutdownController.java | 14 +- .../tez/test/FaultToleranceTestRunner.java | 44 +- .../org/apache/tez/test/MiniTezCluster.java | 51 +- .../RecoveryServiceWithEventHandlingHook.java | 259 ++- .../org/apache/tez/test/SimpleTestDAG.java | 100 +- .../tez/test/SimpleTestDAG3Vertices.java | 16 +- .../test/java/org/apache/tez/test/TestAM.java | 6 +- .../org/apache/tez/test/TestAMRecovery.java | 40 +- .../org/apache/tez/test/TestDAGRecovery.java | 19 +- .../org/apache/tez/test/TestDAGRecovery2.java | 17 +- .../java/org/apache/tez/test/TestDriver.java | 11 +- .../tez/test/TestExceptionPropagation.java | 56 +- .../apache/tez/test/TestFaultTolerance.java | 354 ++-- .../java/org/apache/tez/test/TestInput.java | 117 +- .../org/apache/tez/test/TestLocalMode.java | 16 +- .../apache/tez/test/TestMiniTezCluster.java | 32 +- .../java/org/apache/tez/test/TestOutput.java | 49 +- .../apache/tez/test/TestPipelinedShuffle.java | 11 +- .../org/apache/tez/test/TestProcessor.java | 133 +- .../org/apache/tez/test/TestRecovery.java | 152 +- .../apache/tez/test/TestSecureShuffle.java | 26 +- .../test/TestTaskErrorsUsingLocalMode.java | 5 - .../java/org/apache/tez/test/TestTezJobs.java | 212 +-- .../apache/tez/test/dag/MultiAttemptDAG.java | 26 +- .../tez/test/dag/SimpleReverseVTestDAG.java | 14 +- .../apache/tez/test/dag/SimpleVTestDAG.java | 14 +- .../tez/test/dag/SixLevelsFailingDAG.java | 89 +- .../tez/test/dag/ThreeLevelsFailingDAG.java | 67 +- .../tez/test/dag/TwoLevelsFailingDAG.java | 131 +- .../org/apache/tez/analyzer/Analyzer.java | 1 - .../org/apache/tez/analyzer/CSVResult.java | 13 +- .../tez/analyzer/plugins/AnalyzerDriver.java | 13 +- .../plugins/ContainerReuseAnalyzer.java | 3 +- .../plugins/CriticalPathAnalyzer.java | 131 +- .../analyzer/plugins/DagOverviewAnalyzer.java | 18 +- .../analyzer/plugins/HungTaskAnalyzer.java | 14 +- .../plugins/InputReadErrorAnalyzer.java | 6 +- .../analyzer/plugins/LocalityAnalyzer.java | 18 +- .../plugins/OneOnOneEdgeAnalyzer.java | 9 +- .../analyzer/plugins/ShuffleTimeAnalyzer.java | 14 +- .../tez/analyzer/plugins/SkewAnalyzer.java | 15 +- .../analyzer/plugins/SlowNodeAnalyzer.java | 16 +- .../analyzer/plugins/SlowTaskIdentifier.java | 13 +- .../plugins/SlowestVertexAnalyzer.java | 9 +- .../analyzer/plugins/SpillAnalyzerImpl.java | 5 +- .../plugins/TaskAssignmentAnalyzer.java | 4 +- .../TaskAttemptResultStatisticsAnalyzer.java | 6 +- .../plugins/TaskConcurrencyAnalyzer.java | 25 +- .../tez/analyzer/plugins/TezAnalyzerBase.java | 24 +- .../VertexLevelCriticalPathAnalyzer.java | 7 +- .../apache/tez/analyzer/utils/SVGUtils.java | 166 +- .../org/apache/tez/analyzer/utils/Utils.java | 6 +- .../org/apache/tez/analyzer/TestAnalyzer.java | 226 +-- .../javadoc/doclet/ConfigStandardDoclet.java | 12 +- .../tez/tools/javadoc/model/Config.java | 7 +- .../tools/javadoc/model/ConfigProperty.java | 8 +- .../tez/tools/javadoc/util/HtmlWriter.java | 18 +- .../apache/tez/tools/javadoc/util/Writer.java | 6 +- .../tez/tools/javadoc/util/XmlWriter.java | 9 +- .../org/apache/tez/tools/TFileLoader.java | 4 +- .../apache/tez/tools/TFileRecordReader.java | 20 +- 1096 files changed, 20761 insertions(+), 21530 deletions(-) diff --git a/hadoop-shim-impls/hadoop-shim-2.7/src/main/java/org/apache/tez/hadoop/shim/HadoopShim27.java b/hadoop-shim-impls/hadoop-shim-2.7/src/main/java/org/apache/tez/hadoop/shim/HadoopShim27.java index c5163036a1..3800b02563 100644 --- a/hadoop-shim-impls/hadoop-shim-2.7/src/main/java/org/apache/tez/hadoop/shim/HadoopShim27.java +++ b/hadoop-shim-impls/hadoop-shim-2.7/src/main/java/org/apache/tez/hadoop/shim/HadoopShim27.java @@ -48,5 +48,4 @@ public Set getSupportedResourceTypes(RegisterApplicationMasterResponse r } return supportedTypes; } - } diff --git a/hadoop-shim-impls/hadoop-shim-2.7/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java b/hadoop-shim-impls/hadoop-shim-2.7/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java index c9937f8b94..55863ef26a 100644 --- a/hadoop-shim-impls/hadoop-shim-2.7/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java +++ b/hadoop-shim-impls/hadoop-shim-2.7/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java @@ -77,5 +77,4 @@ public void testLoaderOverrideInvalidVersion() { Assert.assertNotNull(shim); Assert.assertEquals(DefaultHadoopShim.class, shim.getClass()); } - } diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28.java b/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28.java index 5504c02b72..9e76a06747 100644 --- a/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28.java +++ b/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28.java @@ -53,7 +53,7 @@ public Set getSupportedResourceTypes(RegisterApplicationMasterResponse r @Override public FinalApplicationStatus applyFinalApplicationStatusCorrection(FinalApplicationStatus orig, - boolean isSessionMode, boolean isError) { + boolean isSessionMode, boolean isError) { switch (orig) { case FAILED: // App is failed if dag failed in non-session mode or there was an error. diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java b/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java index 353d590e6d..12afefc02a 100644 --- a/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java +++ b/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java @@ -76,5 +76,4 @@ public void testLoaderOverrideInvalidVersion() { Assert.assertNotNull(shim); Assert.assertEquals(DefaultHadoopShim.class, shim.getClass()); } - } diff --git a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShim.java b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShim.java index 47da74a408..fa7475fca6 100644 --- a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShim.java +++ b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShim.java @@ -31,6 +31,7 @@ public abstract class HadoopShim { /** * Set up Hadoop Caller Context + * * @param context Context to be set */ public void setHadoopCallerContext(String context) { @@ -49,6 +50,7 @@ public void clearHadoopCallerContext() { /** * Extract supported Resource types from the RM's response when the AM registers + * * @param response ApplicationMasterResponse from RM after registering * @return Set of Resource types that are supported */ @@ -57,7 +59,7 @@ public Set getSupportedResourceTypes(RegisterApplicationMasterResponse r } public FinalApplicationStatus applyFinalApplicationStatusCorrection(FinalApplicationStatus orig, - boolean isSessionMode, boolean isError) { + boolean isSessionMode, boolean isError) { return orig; } } diff --git a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimProvider.java b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimProvider.java index 728408cef6..fc71cf4e8b 100644 --- a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimProvider.java +++ b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimProvider.java @@ -27,12 +27,12 @@ public abstract class HadoopShimProvider { /** * Provides a HadoopShim if the version matches the implementation of the Shim + * * @param hadoopVersion Full version string of hadoop-common's version - * @param majorVersion Major version - * @param minorVersion Minor version + * @param majorVersion Major version + * @param minorVersion Minor version * @return HadoopShim object if there is a match else null */ public abstract HadoopShim createHadoopShim(String hadoopVersion, int majorVersion, - int minorVersion); - + int minorVersion); } diff --git a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java index 9d9dc3ff55..cf7a37913f 100644 --- a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java +++ b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java @@ -117,7 +117,7 @@ public HadoopShimsLoader(Configuration conf) { try { LOG.debug("Using Reflection to create HadoopShim from provider class=" + overrideProviderClassStr); - Class clazz = (Class)Class.forName( + Class clazz = (Class) Class.forName( overrideProviderClassStr, true, Thread.currentThread().getContextClassLoader()); selectedShimProvider = clazz.newInstance(); selectedShim = selectedShimProvider.createHadoopShim(versionStr, @@ -143,7 +143,6 @@ public HadoopShimsLoader(Configuration conf) { + ", hadoopVersion=" + versionStr + ", majorVersion=" + version.majorVersion + ", minorVersion=" + version.minorVersion); - } public HadoopShim getHadoopShim() { @@ -181,5 +180,4 @@ public Version(String versionString) { } } } - } diff --git a/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShimsLoader.java b/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShimsLoader.java index b0215ed7a3..3ea08ec1b0 100644 --- a/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShimsLoader.java +++ b/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShimsLoader.java @@ -50,5 +50,4 @@ public void testInvalidOverride() { HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); HadoopShim shim = loader.getHadoopShim(); } - } diff --git a/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java b/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java index 1b61c45d05..3f1f74eee5 100644 --- a/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,7 +36,7 @@ class AMConfiguration { private LocalResource binaryConfLRsrc; AMConfiguration(TezConfiguration tezConf, Map localResources, - Credentials credentials) { + Credentials credentials) { this.tezConf = tezConf; if (localResources != null) { addAMLocalResources(localResources); @@ -44,21 +44,20 @@ class AMConfiguration { if (credentials != null) { setCredentials(credentials); } - } void addAMLocalResources(Map localResources) { this.amLocalResources.putAll(localResources); } - + void clearAMLocalResources() { this.amLocalResources.clear(); } - + void setCredentials(Credentials credentials) { this.credentials = credentials; } - + void setTezConfiguration(TezConfiguration tezConf) { this.tezConf = tezConf; } @@ -78,13 +77,12 @@ TezConfiguration getTezConfiguration() { Credentials getCredentials() { return credentials; } - + void setBinaryConfLR(LocalResource binaryConfLRsrc) { this.binaryConfLRsrc = binaryConfLRsrc; } - + LocalResource getBinaryConfLR() { return binaryConfLRsrc; } - } diff --git a/tez-api/src/main/java/org/apache/tez/client/CallerContext.java b/tez-api/src/main/java/org/apache/tez/client/CallerContext.java index f6c67cfe9c..f20164c423 100644 --- a/tez-api/src/main/java/org/apache/tez/client/CallerContext.java +++ b/tez-api/src/main/java/org/apache/tez/client/CallerContext.java @@ -64,28 +64,30 @@ private CallerContext() { /** * Instantiate the Caller Context - * @param context Context in which Tez is being invoked. For example, HIVE or PIG. - * @param callerId Caller ID. An ID to uniquely identifier the caller within the callerType - * namespace + * + * @param context Context in which Tez is being invoked. For example, HIVE or PIG. + * @param callerId Caller ID. An ID to uniquely identifier the caller within the callerType + * namespace * @param callerType Type of the caller. Should ideally be used along with callerId to uniquely * identify the caller. When used with YARN Timeline, this should map to * the Timeline Entity Type. For example, HIVE_QUERY_ID. - * @param blob Free-form text or a json-representation of relevant meta-data. - * This can be used to describe the work being done. For example, for Hive, - * this could be the Hive query text. + * @param blob Free-form text or a json-representation of relevant meta-data. + * This can be used to describe the work being done. For example, for Hive, + * this could be the Hive query text. * @return CallerContext */ public static CallerContext create(String context, String callerId, - String callerType, @Nullable String blob) { + String callerType, @Nullable String blob) { return new CallerContext(context, callerId, callerType, blob); } /** * Instantiate the Caller Context + * * @param context Context in which Tez is being invoked. For example, HIVE or PIG. - * @param blob Free-form text or a json-representation of relevant meta-data. - * This can be used to describe the work being done. For example, for Hive, - * this could be the Hive query text. + * @param blob Free-form text or a json-representation of relevant meta-data. + * This can be used to describe the work being done. For example, for Hive, + * this could be the Hive query text. * @return CallerContext */ @Private @@ -93,9 +95,8 @@ public static CallerContext create(String context, @Nullable String blob) { return new CallerContext(context, blob); } - private CallerContext(String context, String callerId, String callerType, - @Nullable String blob) { + @Nullable String blob) { if (callerId != null || callerType != null) { setCallerIdAndType(callerId, callerType); } @@ -135,15 +136,15 @@ public CallerContext setContext(String context) { } /** - * @param callerId Caller ID. An ID to uniquely identifier the caller within the callerType - * namespace + * @param callerId Caller ID. An ID to uniquely identifier the caller within the callerType + * namespace * @param callerType Type of the caller. Should ideally be used along with callerId to uniquely * identify the caller. When used with YARN Timeline, this should map to * the Timeline Entity Type. For example, HIVE_QUERY_ID. */ public CallerContext setCallerIdAndType(String callerId, String callerType) { Preconditions.checkArgument(callerType != null && !callerType.isEmpty() - && callerId != null && !callerId.isEmpty(), + && callerId != null && !callerId.isEmpty(), "Caller Id and Caller Type cannot be null or empty"); this.callerType = callerType; this.callerId = callerId; @@ -175,6 +176,4 @@ public String contextAsSimpleString() { + ", callerId=" + callerId + " }"; } - - } diff --git a/tez-api/src/main/java/org/apache/tez/client/FrameworkClient.java b/tez-api/src/main/java/org/apache/tez/client/FrameworkClient.java index 2ec6d2864d..742643a2f6 100644 --- a/tez-api/src/main/java/org/apache/tez/client/FrameworkClient.java +++ b/tez-api/src/main/java/org/apache/tez/client/FrameworkClient.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -97,7 +97,7 @@ public abstract ApplicationId submitApplication(ApplicationSubmissionContext app public abstract boolean isRunning() throws IOException; public TezAppMasterStatus getAMStatus(Configuration conf, ApplicationId appId, - UserGroupInformation ugi) throws TezException, ServiceException, IOException { + UserGroupInformation ugi) throws TezException, ServiceException, IOException { DAGClientAMProtocolBlockingPB proxy = getProxy(conf, appId, ugi); if (proxy == null) { @@ -109,7 +109,7 @@ public TezAppMasterStatus getAMStatus(Configuration conf, ApplicationId appId, } public DAGClient submitDag(DAG dag, SubmitDAGRequestProto request, String clientName, - ApplicationId sessionAppId, long clientTimeout, UserGroupInformation ugi, TezConfiguration tezConf) + ApplicationId sessionAppId, long clientTimeout, UserGroupInformation ugi, TezConfiguration tezConf) throws IOException, TezException, DAGSubmissionTimedOut { DAGClientAMProtocolBlockingPB proxy = null; try { @@ -149,7 +149,7 @@ public DAGClient submitDag(DAG dag, SubmitDAGRequestProto request, String client } protected DAGClientAMProtocolBlockingPB waitForProxy(long clientTimeout, Configuration conf, - ApplicationId sessionAppId, UserGroupInformation ugi) + ApplicationId sessionAppId, UserGroupInformation ugi) throws IOException, TezException, InterruptedException { long startTime = System.currentTimeMillis(); long endTime = startTime + (clientTimeout * 1000); @@ -172,7 +172,7 @@ protected DAGClientAMProtocolBlockingPB waitForProxy(long clientTimeout, Configu * that proxy a shutdownSession was called. */ public boolean shutdownSession(Configuration conf, ApplicationId sessionAppId, - UserGroupInformation ugi) throws TezException, IOException, ServiceException { + UserGroupInformation ugi) throws TezException, IOException, ServiceException { DAGClientAMProtocolBlockingPB proxy = getProxy(conf, sessionAppId, ugi); if (proxy != null) { ShutdownSessionRequestProto request = ShutdownSessionRequestProto.newBuilder().build(); @@ -183,7 +183,7 @@ public boolean shutdownSession(Configuration conf, ApplicationId sessionAppId, } protected DAGClientAMProtocolBlockingPB getProxy(Configuration conf, ApplicationId sessionAppId, - UserGroupInformation ugi) throws TezException, IOException { + UserGroupInformation ugi) throws TezException, IOException { return TezClientUtils.getAMProxy(this, conf, sessionAppId, ugi); } } diff --git a/tez-api/src/main/java/org/apache/tez/client/TezApiVersionInfo.java b/tez-api/src/main/java/org/apache/tez/client/TezApiVersionInfo.java index 2870ae43b0..19fda27ebc 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezApiVersionInfo.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezApiVersionInfo.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,5 +25,4 @@ public class TezApiVersionInfo extends VersionInfo { public TezApiVersionInfo() { super("tez-api"); } - } diff --git a/tez-api/src/main/java/org/apache/tez/client/TezAppMasterStatus.java b/tez-api/src/main/java/org/apache/tez/client/TezAppMasterStatus.java index d24dbf0486..54334a99d6 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezAppMasterStatus.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezAppMasterStatus.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java index c37f0c181d..d3423299a2 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -110,7 +110,7 @@ public class TezClient { private static final String appIdStrPrefix = "application"; private static final String APPLICATION_ID_PREFIX = appIdStrPrefix + '_'; private static final long PREWARM_WAIT_MS = 500; - + @VisibleForTesting static final String NO_CLUSTER_DIAGNOSTICS_MSG = "No cluster diagnostics found."; @@ -164,7 +164,7 @@ private TezClient(String name, TezConfiguration tezConf) { @Nullable Map localResources, @Nullable Credentials credentials) { this(name, tezConf, tezConf.getBoolean( - TezConfiguration.TEZ_AM_SESSION_MODE, TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT), + TezConfiguration.TEZ_AM_SESSION_MODE, TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT), localResources, credentials); } @@ -181,8 +181,8 @@ protected TezClient(String name, TezConfiguration tezConf, boolean isSession, @Private protected TezClient(String name, TezConfiguration tezConf, boolean isSession, - @Nullable Map localResources, - @Nullable Credentials credentials, ServicePluginsDescriptor servicePluginsDescriptor) { + @Nullable Map localResources, + @Nullable Credentials credentials, ServicePluginsDescriptor servicePluginsDescriptor) { this.clientName = name; this.isSession = isSession; // Set in conf for local mode AM to figure out whether in session mode or not @@ -204,13 +204,12 @@ protected TezClient(String name, TezConfiguration tezConf, boolean isSession, this.maxSubmitDAGRequestSizeThroughIPC = tezConf.getInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH_DEFAULT) - tezConf.getInt(TezConfiguration.TEZ_IPC_PAYLOAD_RESERVED_BYTES, - TezConfiguration.TEZ_IPC_PAYLOAD_RESERVED_BYTES_DEFAULT); + TezConfiguration.TEZ_IPC_PAYLOAD_RESERVED_BYTES_DEFAULT); Limits.setConfiguration(tezConf); LOG.info("Tez Client Version: " + apiVersionInfo.toString()); } - /** * Create a new TezClientBuilder. This can be used to setup additional parameters * like session mode, local resources, credentials, servicePlugins, etc. @@ -322,7 +321,7 @@ public static TezClient create(String name, TezConfiguration tezConf, boolean is * accumulate across DAG submissions and are never removed from the classpath. * Only LocalResourceType.FILE is supported. All files will be treated as * private. - * + * * @param localFiles the files to be made available in the AM */ public synchronized void addAppMasterLocalFiles(Map localFiles) { @@ -332,7 +331,7 @@ public synchronized void addAppMasterLocalFiles(Map local } amConfig.addAMLocalResources(localFiles); } - + /** * If the next DAG App Master needs different local files, then use this * method to clear the local files and then add the new local files @@ -342,7 +341,7 @@ public synchronized void addAppMasterLocalFiles(Map local public synchronized void clearAppMasterLocalFiles() { amConfig.clearAMLocalResources(); } - + /** * Set security credentials to be used inside the app master, if needed. Tez App * Master needs credentials to access the staging directory and for most HDFS @@ -352,7 +351,7 @@ public synchronized void clearAppMasterLocalFiles() { * credentials must be supplied by the user. These will be used by the App * Master for the next DAG.
In session mode, credentials, if needed, must be * set before calling start() - * + * * @param credentials credentials */ public synchronized void setAppMasterCredentials(Credentials credentials) { @@ -388,16 +387,16 @@ public synchronized void start() throws TezException, IOException { LOG.info("Session mode. Starting session."); TezClientUtils.processTezLocalCredentialsFile(sessionCredentials, amConfig.getTezConfiguration()); - + clientTimeout = amConfig.getTezConfiguration().getInt( TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS, TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS_DEFAULT); - + try { if (sessionAppId == null) { sessionAppId = createApplication(); } - + ApplicationSubmissionContext appContext = setupApplicationContext(); frameworkClient.submitApplication(appContext); ApplicationReport appReport = frameworkClient.getApplicationReport(sessionAppId); @@ -461,8 +460,8 @@ public synchronized TezClient getClient(ApplicationId appId) throws TezException LOG.info("Session mode. Reconnecting to session: " + sessionAppId.toString()); clientTimeout = amConfig.getTezConfiguration().getInt( - TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS, - TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS_DEFAULT); + TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS, + TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS_DEFAULT); try { setupApplicationContext(); @@ -490,24 +489,24 @@ private void startFrameworkClient() { private ApplicationSubmissionContext setupApplicationContext() throws IOException, YarnException { TezClientUtils.processTezLocalCredentialsFile(sessionCredentials, - amConfig.getTezConfiguration()); + amConfig.getTezConfiguration()); Map tezJarResources = getTezJarResources(sessionCredentials); // Add session token for shuffle TezClientUtils.createSessionToken(sessionAppId.toString(), - jobTokenSecretManager, sessionCredentials); + jobTokenSecretManager, sessionCredentials); ApplicationSubmissionContext appContext = - TezClientUtils.createApplicationSubmissionContext( - sessionAppId, - null, clientName, amConfig, - tezJarResources, sessionCredentials, usingTezArchiveDeploy, apiVersionInfo, - servicePluginsDescriptor, javaOptsChecker); + TezClientUtils.createApplicationSubmissionContext( + sessionAppId, + null, clientName, amConfig, + tezJarResources, sessionCredentials, usingTezArchiveDeploy, apiVersionInfo, + servicePluginsDescriptor, javaOptsChecker); // Set Tez Sessions to not retry on AM crashes if recovery is disabled if (!amConfig.getTezConfiguration().getBoolean( - TezConfiguration.DAG_RECOVERY_ENABLED, - TezConfiguration.DAG_RECOVERY_ENABLED_DEFAULT)) { + TezConfiguration.DAG_RECOVERY_ENABLED, + TezConfiguration.DAG_RECOVERY_ENABLED_DEFAULT)) { appContext.setMaxAppAttempts(1); } return appContext; @@ -515,41 +514,40 @@ private ApplicationSubmissionContext setupApplicationContext() throws IOExceptio private void setupJavaOptsChecker() { if (this.amConfig.getTezConfiguration().getBoolean( - TezConfiguration.TEZ_CLIENT_JAVA_OPTS_CHECKER_ENABLED, - TezConfiguration.TEZ_CLIENT_JAVA_OPTS_CHECKER_ENABLED_DEFAULT)) { + TezConfiguration.TEZ_CLIENT_JAVA_OPTS_CHECKER_ENABLED, + TezConfiguration.TEZ_CLIENT_JAVA_OPTS_CHECKER_ENABLED_DEFAULT)) { String javaOptsCheckerClassName = this.amConfig.getTezConfiguration().get( - TezConfiguration.TEZ_CLIENT_JAVA_OPTS_CHECKER_CLASS, ""); + TezConfiguration.TEZ_CLIENT_JAVA_OPTS_CHECKER_CLASS, ""); if (!javaOptsCheckerClassName.isEmpty()) { try { javaOptsChecker = ReflectionUtils.createClazzInstance(javaOptsCheckerClassName); } catch (Exception e) { LOG.warn("Failed to initialize configured Java Opts Checker" - + " (" + TezConfiguration.TEZ_CLIENT_JAVA_OPTS_CHECKER_CLASS - + ") , checkerClass=" + javaOptsCheckerClassName - + ". Disabling checker.", e); + + " (" + TezConfiguration.TEZ_CLIENT_JAVA_OPTS_CHECKER_CLASS + + ") , checkerClass=" + javaOptsCheckerClassName + + ". Disabling checker.", e); javaOptsChecker = null; } } else { javaOptsChecker = new JavaOptsChecker(); } - } } private void startClientHeartbeat() { long amClientKeepAliveTimeoutIntervalMillis = - TezCommonUtils.getAMClientHeartBeatTimeoutMillis(amConfig.getTezConfiguration()); + TezCommonUtils.getAMClientHeartBeatTimeoutMillis(amConfig.getTezConfiguration()); // Poll at minimum of 1 second interval long pollPeriod = TezCommonUtils. - getAMClientHeartBeatPollIntervalMillis(amConfig.getTezConfiguration(), - amClientKeepAliveTimeoutIntervalMillis, 10); + getAMClientHeartBeatPollIntervalMillis(amConfig.getTezConfiguration(), + amClientKeepAliveTimeoutIntervalMillis, 10); boolean isLocal = amConfig.getTezConfiguration().getBoolean( - TezConfiguration.TEZ_LOCAL_MODE, TezConfiguration.TEZ_LOCAL_MODE_DEFAULT); + TezConfiguration.TEZ_LOCAL_MODE, TezConfiguration.TEZ_LOCAL_MODE_DEFAULT); if (!isLocal && amClientKeepAliveTimeoutIntervalMillis > 0) { amKeepAliveService = Executors.newSingleThreadScheduledExecutor( - new ThreadFactoryBuilder() - .setDaemon(true).setNameFormat("AMKeepAliveThread #%d").build()); + new ThreadFactoryBuilder() + .setDaemon(true).setNameFormat("AMKeepAliveThread #%d").build()); amKeepAliveService.scheduleWithFixedDelay(new Runnable() { private DAGClientAMProtocolBlockingPB proxy; @@ -598,7 +596,7 @@ public DAGClientAMProtocolBlockingPB sendAMHeartbeat(DAGClientAMProtocolBlocking * cluster.
In session mode, it submits the DAG to the session App Master. It * blocks until either the DAG is submitted to the session or configured * timeout period expires. Cleans up session if the submission timed out. - * + * * @param dag * DAG to be submitted to Session * @return DAGClient to monitor the DAG @@ -606,7 +604,7 @@ public DAGClientAMProtocolBlockingPB sendAMHeartbeat(DAGClientAMProtocolBlocking * @throws IOException * @throws DAGSubmissionTimedOut * if submission timed out - */ + */ public synchronized DAGClient submitDAG(DAG dag) throws TezException, IOException { DAGClient result = isSession ? submitDAGSession(dag) : submitDAGApplication(dag); if (result != null) { @@ -625,8 +623,7 @@ private void killAndClosePrewarmDagClient(long waitTimeMs) { LOG.info("Waiting for prewarm DAG to shut down"); prewarmDagClient.waitForCompletion(waitTimeMs); } - } - catch (Exception ex) { + } catch (Exception ex) { LOG.warn("Failed to shut down the prewarm DAG " + prewarmDagClient, ex); } closePrewarmDagClient(); @@ -643,12 +640,12 @@ private void closePrewarmDagClient() { } prewarmDagClient = null; } - + private DAGClient submitDAGSession(DAG dag) throws TezException, IOException { - Preconditions.checkState(isSession == true, - "submitDAG with additional resources applies to only session mode. " + - "In non-session mode please specify all resources in the initial configuration"); - + Preconditions.checkState(isSession == true, + "submitDAG with additional resources applies to only session mode. " + + "In non-session mode please specify all resources in the initial configuration"); + verifySessionStateForSubmission(); String callerContextStr = ""; @@ -656,11 +653,11 @@ private DAGClient submitDAGSession(DAG dag) throws TezException, IOException { callerContextStr = ", callerContext=" + dag.getCallerContext().contextAsSimpleString(); } LOG.info("Submitting dag to TezSession" - + ", sessionName=" + clientName - + ", applicationId=" + sessionAppId - + ", dagName=" + dag.getName() - + callerContextStr); - + + ", sessionName=" + clientName + + ", applicationId=" + sessionAppId + + ", dagName=" + dag.getName() + + callerContextStr); + if (!additionalLocalResources.isEmpty()) { for (LocalResource lr : additionalLocalResources.values()) { Preconditions.checkArgument(lr.getType() == LocalResourceType.FILE, "LocalResourceType: " @@ -678,7 +675,7 @@ private DAGClient submitDAGSession(DAG dag) throws TezException, IOException { requestBuilder.setAdditionalAmResources(DagTypeConverters .convertFromLocalResources(additionalLocalResources)); } - + additionalLocalResources.clear(); // if request size exceeds maxSubmitDAGRequestSizeThroughIPC, we serialize them to HDFS @@ -792,6 +789,7 @@ public synchronized void stop() throws TezException, IOException { } } } + private boolean isJobInTerminalState(YarnApplicationState yarnApplicationState) { return (yarnApplicationState == YarnApplicationState.FINISHED || yarnApplicationState == YarnApplicationState.FAILED @@ -805,7 +803,7 @@ private boolean isJobInTerminalState(YarnApplicationState yarnApplicationState) public String getClientName() { return clientName; } - + @Private @VisibleForTesting public synchronized ApplicationId getAppMasterApplicationId() { @@ -820,7 +818,7 @@ public synchronized ApplicationId getAppMasterApplicationId() { * Get the status of the App Master executing the DAG * In non-session mode it returns the status of the last submitted DAG App Master * In session mode, it returns the status of the App Master hosting the session - * + * * @return State of the session * @throws TezException * @throws IOException @@ -840,28 +838,28 @@ public synchronized TezAppMasterStatus getAppMasterStatus() throws TezException, ApplicationReport appReport = frameworkClient.getApplicationReport( appId); switch (appReport.getYarnApplicationState()) { - case NEW: - case NEW_SAVING: - case ACCEPTED: - case SUBMITTED: - return TezAppMasterStatus.INITIALIZING; - case FAILED: - case KILLED: - diagnostics = appReport.getDiagnostics(); - LOG.info("App did not succeed. Diagnostics: " - + (appReport.getDiagnostics() != null ? appReport.getDiagnostics() - : NO_CLUSTER_DIAGNOSTICS_MSG)); - return TezAppMasterStatus.SHUTDOWN; - case FINISHED: - return TezAppMasterStatus.SHUTDOWN; - case RUNNING: - try { - return frameworkClient.getAMStatus(amConfig.getTezConfiguration(), appId, getUgi()); - } catch (TezException e) { - LOG.info("Failed to retrieve AM Status via proxy", e); - } catch (ServiceException e) { - LOG.info("Failed to retrieve AM Status via proxy", e); - } + case NEW: + case NEW_SAVING: + case ACCEPTED: + case SUBMITTED: + return TezAppMasterStatus.INITIALIZING; + case FAILED: + case KILLED: + diagnostics = appReport.getDiagnostics(); + LOG.info("App did not succeed. Diagnostics: " + + (appReport.getDiagnostics() != null ? appReport.getDiagnostics() + : NO_CLUSTER_DIAGNOSTICS_MSG)); + return TezAppMasterStatus.SHUTDOWN; + case FINISHED: + return TezAppMasterStatus.SHUTDOWN; + case RUNNING: + try { + return frameworkClient.getAMStatus(amConfig.getTezConfiguration(), appId, getUgi()); + } catch (TezException e) { + LOG.info("Failed to retrieve AM Status via proxy", e); + } catch (ServiceException e) { + LOG.info("Failed to retrieve AM Status via proxy", e); + } } } catch (ApplicationNotFoundException e) { return TezAppMasterStatus.SHUTDOWN; @@ -870,7 +868,7 @@ public synchronized TezAppMasterStatus getAppMasterStatus() throws TezException, } return TezAppMasterStatus.INITIALIZING; } - + /** * API to help pre-allocate containers in session mode. In non-session mode * this is ignored. The pre-allocated containers may be re-used by subsequent @@ -919,7 +917,7 @@ public synchronized void preWarm(PreWarmVertex preWarmVertex) throws TezExceptio */ @Unstable public synchronized void preWarm(PreWarmVertex preWarmVertex, - long timeout, TimeUnit unit) + long timeout, TimeUnit unit) throws TezException, IOException { if (!isSession) { // do nothing for non session mode. This is there to let the code @@ -930,7 +928,7 @@ public synchronized void preWarm(PreWarmVertex preWarmVertex, } verifySessionStateForSubmission(); - + DAG dag = org.apache.tez.dag.api.DAG.create(TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX + "_" + preWarmDAGCounter++); dag.addVertex(preWarmVertex); @@ -942,14 +940,13 @@ public synchronized void preWarm(PreWarmVertex preWarmVertex, throw new IOException("Interrupted while waiting for AM to become " + "available", e); } - if(isReady) { + if (isReady) { prewarmDagClient = submitDAG(dag); } else { throw new SessionNotReady("Tez AM not ready, could not submit DAG"); } } - /** * Wait till the DAG is ready to be submitted. * In non-session mode this is a no-op since the application can be immediately @@ -957,7 +954,7 @@ public synchronized void preWarm(PreWarmVertex preWarmVertex, * In session mode, this waits for the session host to be ready to accept a DAG * @throws IOException * @throws TezException - * @throws InterruptedException + * @throws InterruptedException */ @Evolving public synchronized void waitTillReady() throws IOException, TezException, InterruptedException { @@ -1028,6 +1025,7 @@ private void waitNonSessionTillReady() throws IOException, TezException { } } } + @VisibleForTesting // for testing @Private @@ -1043,16 +1041,17 @@ private void verifySessionStateForSubmission() throws SessionNotRunning { throw new SessionNotRunning("Session stopped by user"); } } - + private DAGClient submitDAGApplication(DAG dag) throws TezException, IOException { ApplicationId appId = createApplication(); return submitDAGApplication(appId, dag); } - @Private // To be used only by YarnRunner + @Private + // To be used only by YarnRunner DAGClient submitDAGApplication(ApplicationId appId, DAG dag) - throws TezException, IOException { + throws TezException, IOException { LOG.info("Submitting DAG application with id: " + appId); try { // Use the AMCredentials object in client mode, since this won't be re-used. @@ -1116,13 +1115,13 @@ private synchronized Map getTezJarResources(Credentials c @Private static DAGClient getDAGClient(ApplicationId appId, TezConfiguration tezConf, - FrameworkClient frameworkClient, UserGroupInformation ugi) throws IOException, TezException { + FrameworkClient frameworkClient, UserGroupInformation ugi) throws IOException, TezException { return new DAGClientImpl(appId, getDefaultTezDAGID(appId), tezConf, frameworkClient, ugi); } @Private // Used only for MapReduce compatibility code static DAGClient getDAGClient(ApplicationId appId, TezConfiguration tezConf, - FrameworkClient frameworkClient) throws IOException, TezException { + FrameworkClient frameworkClient) throws IOException, TezException { UserGroupInformation ugi = UserGroupInformation.createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()); return getDAGClient(appId, tezConf, frameworkClient, ugi); @@ -1153,12 +1152,12 @@ public NumberFormat initialValue() { // Used only for MapReduce compatibility code private static String getDefaultTezDAGID(ApplicationId applicationId) { - return (new StringBuilder(DAG)).append(SEPARATOR). - append(applicationId.getClusterTimestamp()). - append(SEPARATOR). - append(tezAppIdFormat.get().format(applicationId.getId())). - append(SEPARATOR). - append(tezDagIdFormat.get().format(1)).toString(); + return (new StringBuilder(DAG)).append(SEPARATOR). + append(applicationId.getClusterTimestamp()). + append(SEPARATOR). + append(tezAppIdFormat.get().format(applicationId.getId())). + append(SEPARATOR). + append(tezDagIdFormat.get().format(1)).toString(); } @VisibleForTesting @@ -1276,15 +1275,15 @@ public TezClient build() { public static ApplicationId appIdfromString(String appIdStr) { if (!appIdStr.startsWith(APPLICATION_ID_PREFIX)) { throw new IllegalArgumentException("Invalid ApplicationId prefix: " - + appIdStr + ". The valid ApplicationId should start with prefix " - + appIdStrPrefix); + + appIdStr + ". The valid ApplicationId should start with prefix " + + appIdStrPrefix); } try { int pos1 = APPLICATION_ID_PREFIX.length() - 1; int pos2 = appIdStr.indexOf('_', pos1 + 1); if (pos2 < 0) { throw new IllegalArgumentException("Invalid ApplicationId: " - + appIdStr); + + appIdStr); } long rmId = Long.parseLong(appIdStr.substring(pos1 + 1, pos2)); int appId = Integer.parseInt(appIdStr.substring(pos2 + 1)); @@ -1292,7 +1291,7 @@ public static ApplicationId appIdfromString(String appIdStr) { return applicationId; } catch (NumberFormatException n) { throw new IllegalArgumentException("Invalid ApplicationId: " - + appIdStr, n); + + appIdStr, n); } } } diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java index d0fc374a81..f80f0096b6 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -136,7 +136,7 @@ private static FileStatus[] getLRFileStatus(String fileName, Configuration conf) FileSystem fs = p.getFileSystem(conf); p = fs.resolvePath(p.makeQualified(fs.getUri(), fs.getWorkingDirectory())); - FileSystem targetFS = p.getFileSystem(conf); + FileSystem targetFS = p.getFileSystem(conf); if (targetFS.isDirectory(p)) { return targetFS.listStatus(p); } else { @@ -147,7 +147,7 @@ private static FileStatus[] getLRFileStatus(String fileName, Configuration conf) /** * Setup LocalResource map for Tez jars based on provided Configuration - * + * * @param conf * Configuration to use to access Tez jars' locations * @param credentials @@ -158,14 +158,14 @@ private static FileStatus[] getLRFileStatus(String fileName, Configuration conf) * @throws IOException */ static boolean setupTezJarsLocalResources(TezConfiguration conf, - Credentials credentials, Map tezJarResources) + Credentials credentials, Map tezJarResources) throws IOException { Objects.requireNonNull(credentials, "A non-null credentials object should be specified"); boolean usingTezArchive = false; - if (conf.getBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, false)){ - LOG.info("Ignoring '" + TezConfiguration.TEZ_LIB_URIS + "' since '" + - TezConfiguration.TEZ_IGNORE_LIB_URIS + "' is set to true"); + if (conf.getBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, false)) { + LOG.info("Ignoring '" + TezConfiguration.TEZ_LIB_URIS + "' since '" + + TezConfiguration.TEZ_IGNORE_LIB_URIS + "' is set to true"); } else { // Add tez jars to local resource String[] tezJarUris = conf.getStrings(TezConfiguration.TEZ_LIB_URIS); @@ -182,7 +182,7 @@ static boolean setupTezJarsLocalResources(TezConfiguration conf, + conf.get(TezConfiguration.TEZ_LIB_URIS_CLASSPATH)); usingTezArchive = addLocalResources(conf, tezJarUris, - tezJarResources, credentials); + tezJarResources, credentials); if (tezJarResources.isEmpty()) { throw new TezUncheckedException( @@ -200,8 +200,8 @@ static boolean setupTezJarsLocalResources(TezConfiguration conf, } private static boolean addLocalResources(Configuration conf, - String[] configUris, Map tezJarResources, - Credentials credentials) throws IOException { + String[] configUris, Map tezJarResources, + Credentials credentials) throws IOException { boolean usingTezArchive = false; if (configUris == null || configUris.length == 0) { return usingTezArchive; @@ -222,16 +222,16 @@ private static boolean addLocalResources(Configuration conf, LocalResourceType type = null; //Check if path is an archive - if(p.getName().endsWith(".tar.gz") || - p.getName().endsWith(".tgz") || - p.getName().endsWith(".zip") || - p.getName().endsWith(".tar")) { - type = LocalResourceType.ARCHIVE; - } else { - type = LocalResourceType.FILE; - } + if (p.getName().endsWith(".tar.gz") || + p.getName().endsWith(".tgz") || + p.getName().endsWith(".zip") || + p.getName().endsWith(".tar")) { + type = LocalResourceType.ARCHIVE; + } else { + type = LocalResourceType.FILE; + } - FileStatus [] fileStatuses = getLRFileStatus(configUri, conf); + FileStatus[] fileStatuses = getLRFileStatus(configUri, conf); for (FileStatus fStatus : fileStatuses) { String linkName; @@ -240,13 +240,13 @@ private static boolean addLocalResources(Configuration conf, continue; } // If the resource is an archive, we've already done this work - if(type != LocalResourceType.ARCHIVE) { + if (type != LocalResourceType.ARCHIVE) { u = fStatus.getPath().toUri(); p = new Path(u); remoteFS = p.getFileSystem(conf); p = remoteFS.resolvePath(p.makeQualified(remoteFS.getUri(), remoteFS.getWorkingDirectory())); - if(null != u.getFragment()) { + if (null != u.getFragment()) { LOG.warn("Fragment set for link being interpreted as a file," + "URI: " + u.toString()); } @@ -254,8 +254,8 @@ private static boolean addLocalResources(Configuration conf, // Add URI fragment or just the filename Path name = new Path((null == u.getFragment()) - ? p.getName() - : u.getFragment()); + ? p.getName() + : u.getFragment()); if (name.isAbsolute()) { throw new IllegalArgumentException("Resource name must be " + "relative, not absolute: " + name @@ -266,7 +266,7 @@ private static boolean addLocalResources(Configuration conf, linkName = name.toUri().getPath(); // For legacy reasons, set archive to tezlib if there is // only a single archive and no fragment - if(type == LocalResourceType.ARCHIVE && + if (type == LocalResourceType.ARCHIVE && configUris.length == 1 && null == u.getFragment()) { linkName = TezConstants.TEZ_TAR_LR_NAME; usingTezArchive = true; @@ -325,7 +325,7 @@ static void processTezLocalCredentialsFile(Credentials credentials, Configuratio * @throws IOException */ public static FileSystem ensureStagingDirExists(Configuration conf, - Path stagingArea) + Path stagingArea) throws IOException { FileSystem fs = stagingArea.getFileSystem(conf); String realUser; @@ -354,7 +354,7 @@ public static FileSystem ensureStagingDirExists(Configuration conf, } return fs; } - + /** * Populate {@link Credentials} for the URI's to access them from their {@link FileSystem}s * @param uris URIs that need to be accessed @@ -363,7 +363,7 @@ public static FileSystem ensureStagingDirExists(Configuration conf, * @throws IOException */ public static void addFileSystemCredentialsFromURIs(Collection uris, Credentials credentials, - Configuration conf) throws IOException { + Configuration conf) throws IOException { // Obtain Credentials for any paths that the user may have configured. if (uris != null && !uris.isEmpty()) { Iterator pathIter = Iterators.transform(uris.iterator(), new Function() { @@ -382,7 +382,7 @@ public Path apply(URI input) { * Obtains tokens for the DAG based on the list of URIs setup in the DAG. The * fetched credentials are populated back into the DAG and can be retrieved * via dag.getCredentials - * + * * @param dag * the dag for which credentials need to be setup * @param sessionCredentials @@ -393,24 +393,24 @@ public Path apply(URI input) { */ @Private static Credentials setupDAGCredentials(DAG dag, Credentials sessionCredentials, - Configuration conf) throws IOException { + Configuration conf) throws IOException { Objects.requireNonNull(sessionCredentials); TezCommonUtils.logCredentials(LOG, sessionCredentials, "session"); Credentials dagCredentials = new Credentials(); // All session creds are required for the DAG. dagCredentials.mergeAll(sessionCredentials); - + // Add additional credentials based on any URIs that the user may have specified. - + // Obtain Credentials for any paths that the user may have configured. addFileSystemCredentialsFromURIs(dag.getURIsForCredentials(), dagCredentials, conf); // Obtain Credentials for the local resources configured on the DAG try { Set lrPaths = new HashSet(); - for (Vertex v: dag.getVertices()) { - for (LocalResource lr: v.getTaskLocalFiles().values()) { + for (Vertex v : dag.getVertices()) { + for (LocalResource lr : v.getTaskLocalFiles().values()) { lrPaths.add(ConverterUtils.getPathFromYarnURL(lr.getResource())); } List dataSources = v.getDataSources(); @@ -422,18 +422,17 @@ static Credentials setupDAGCredentials(DAG dag, Credentials sessionCredentials, addFileSystemCredentialsFromURIs(dataSink.getURIsForCredentials(), dagCredentials, conf); } } - - for (LocalResource lr: dag.getTaskLocalFiles().values()) { + + for (LocalResource lr : dag.getTaskLocalFiles().values()) { lrPaths.add(ConverterUtils.getPathFromYarnURL(lr.getResource())); } Path[] paths = lrPaths.toArray(new Path[lrPaths.size()]); TokenCache.obtainTokensForFileSystems(dagCredentials, paths, conf); - } catch (URISyntaxException e) { throw new IOException(e); } - + return dagCredentials; } @@ -505,7 +504,6 @@ public static ApplicationSubmissionContext createApplicationSubmissionContext( String amLogLevel = amLogParams[0]; maybeAddDefaultLoggingJavaOpts(amLogLevel, vargs); - // FIX sun bug mentioned in TEZ-327 vargs.add("-Dsun.nio.ch.bugLevel=''"); @@ -519,7 +517,6 @@ public static ApplicationSubmissionContext createApplicationSubmissionContext( vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + File.separator + ApplicationConstants.STDERR); - Vector vargsFinal = new Vector(8); // Final command StringBuilder mergedCommand = new StringBuilder(); @@ -561,9 +558,9 @@ public static ApplicationSubmissionContext createApplicationSubmissionContext( // emit conf as PB file // don't overwrite existing conf, needed for TezClient.getClient() so existing containers have stable resource fingerprints - if(!binaryConfPath.getFileSystem(tezConf).exists(binaryConfPath)) { + if (!binaryConfPath.getFileSystem(tezConf).exists(binaryConfPath)) { ConfigurationProto finalConfProto = createFinalConfProtoForApp(tezConf, - servicePluginsDescriptor); + servicePluginsDescriptor); FSDataOutputStream amConfPBOutBinaryStream = null; try { @@ -594,7 +591,7 @@ public static ApplicationSubmissionContext createApplicationSubmissionContext( if (amLocalResources != null && !amLocalResources.isEmpty()) { amResourceProto = DagTypeConverters.convertFromLocalResources(amLocalResources); } else { - amResourceProto = DAGProtos.PlanLocalResourcesProto.getDefaultInstance(); + amResourceProto = DAGProtos.PlanLocalResourcesProto.getDefaultInstance(); } amResourceProto.writeDelimitedTo(sessionJarsPBOutStream); } finally { @@ -604,19 +601,19 @@ public static ApplicationSubmissionContext createApplicationSubmissionContext( } LocalResource sessionJarsPBLRsrc = - TezClientUtils.createLocalResource(fs, - sessionJarsPath, LocalResourceType.FILE, - LocalResourceVisibility.APPLICATION); + TezClientUtils.createLocalResource(fs, + sessionJarsPath, LocalResourceType.FILE, + LocalResourceVisibility.APPLICATION); amLocalResources.put( - TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME, - sessionJarsPBLRsrc); + TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME, + sessionJarsPBLRsrc); String user = UserGroupInformation.getCurrentUser().getShortUserName(); ACLManager aclManager = new ACLManager(user, amConfig.getTezConfiguration()); Map acls = aclManager.toYARNACls(); - if(dag != null) { - + if (dag != null) { + DAGPlan dagPB = prepareAndCreateDAGPlan(dag, amConfig, tezJarResources, tezLrsAsArchive, sessionCreds, servicePluginsDescriptor, javaOptsChecker); @@ -635,15 +632,15 @@ public static ApplicationSubmissionContext createApplicationSubmissionContext( dagPBOutBinaryStream = TezCommonUtils.createFileForAM(fs, binaryPath); dagPB.writeTo(dagPBOutBinaryStream); } finally { - if(dagPBOutBinaryStream != null){ + if (dagPBOutBinaryStream != null) { dagPBOutBinaryStream.close(); } } amLocalResources.put(TezConstants.TEZ_PB_PLAN_BINARY_NAME, - TezClientUtils.createLocalResource(fs, - binaryPath, LocalResourceType.FILE, - LocalResourceVisibility.APPLICATION)); + TezClientUtils.createLocalResource(fs, + binaryPath, LocalResourceType.FILE, + LocalResourceVisibility.APPLICATION)); if (Level.DEBUG.isGreaterOrEqual(Level.toLevel(amLogLevel))) { Path textPath = localizeDagPlanAsText(dagPB, fs, amConfig, strAppId, tezSysStagingPath); @@ -674,7 +671,7 @@ public static ApplicationSubmissionContext createApplicationSubmissionContext( Collection tagsFromConf = amConfig.getTezConfiguration().getTrimmedStringCollection( - TezConfiguration.TEZ_APPLICATION_TAGS); + TezConfiguration.TEZ_APPLICATION_TAGS); appContext.setApplicationType(TezConstants.TEZ_APPLICATION_TYPE); if (tagsFromConf != null && !tagsFromConf.isEmpty()) { @@ -695,15 +692,14 @@ public static ApplicationSubmissionContext createApplicationSubmissionContext( appContext.setAMContainerSpec(amContainer); appContext.setMaxAppAttempts( - amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, - TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS_DEFAULT)); + amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, + TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS_DEFAULT)); return appContext; - } static Credentials prepareAmLaunchCredentials(AMConfiguration amConfig, Credentials sessionCreds, - TezConfiguration conf, Path binaryConfPath) throws IOException { + TezConfiguration conf, Path binaryConfPath) throws IOException { // Setup security tokens Credentials amLaunchCredentials = new Credentials(); @@ -712,12 +708,12 @@ static Credentials prepareAmLaunchCredentials(AMConfiguration amConfig, Credenti String simpleHistoryLogDir = conf.get(TezConfiguration.TEZ_SIMPLE_HISTORY_LOGGING_DIR); if (simpleHistoryLogDir != null && !simpleHistoryLogDir.isEmpty()) { Path simpleHistoryLogDirPath = new Path(simpleHistoryLogDir); - TokenCache.obtainTokensForFileSystems(sessionCreds, new Path[] { simpleHistoryLogDirPath }, + TokenCache.obtainTokensForFileSystems(sessionCreds, new Path[]{simpleHistoryLogDirPath}, conf); } // Add Staging dir creds to the list of session credentials. - TokenCache.obtainTokensForFileSystems(sessionCreds, new Path[] {binaryConfPath }, conf); + TokenCache.obtainTokensForFileSystems(sessionCreds, new Path[]{binaryConfPath}, conf); populateTokenCache(conf, sessionCreds); @@ -733,31 +729,31 @@ static Credentials prepareAmLaunchCredentials(AMConfiguration amConfig, Credenti //get secret keys and tokens and store them into TokenCache private static void populateTokenCache(TezConfiguration conf, Credentials credentials) - throws IOException{ + throws IOException { // add the delegation tokens from configuration String[] nameNodes = conf.getStrings(TezConfiguration.TEZ_JOB_FS_SERVERS); LOG.debug("adding the following namenodes' delegation tokens:" + - Arrays.toString(nameNodes)); - if(nameNodes != null) { + Arrays.toString(nameNodes)); + if (nameNodes != null) { Path[] ps = new Path[nameNodes.length]; - for(int i = 0; i < nameNodes.length; i++) { + for (int i = 0; i < nameNodes.length; i++) { ps[i] = new Path(nameNodes[i]); } TokenCache.obtainTokensForFileSystems(credentials, ps, conf); } } - + static DAGPlan prepareAndCreateDAGPlan(DAG dag, AMConfiguration amConfig, - Map tezJarResources, boolean tezLrsAsArchive, - Credentials credentials, ServicePluginsDescriptor servicePluginsDescriptor, - JavaOptsChecker javaOptsChecker) throws IOException { + Map tezJarResources, boolean tezLrsAsArchive, + Credentials credentials, ServicePluginsDescriptor servicePluginsDescriptor, + JavaOptsChecker javaOptsChecker) throws IOException { Credentials dagCredentials = setupDAGCredentials(dag, credentials, amConfig.getTezConfiguration()); TezCommonUtils.logCredentials(LOG, dagCredentials, "dagPlan"); return dag.createDag(amConfig.getTezConfiguration(), dagCredentials, tezJarResources, amConfig.getBinaryConfLR(), tezLrsAsArchive, servicePluginsDescriptor, javaOptsChecker); } - + static void maybeAddDefaultLoggingJavaOpts(String logLevel, List vargs) { Objects.requireNonNull(vargs); TezClientUtils.addLog4jSystemProperties(logLevel, vargs); @@ -786,7 +782,7 @@ public static String addDefaultsToTaskLaunchCmdOpts(String vOpts, Configuration @Private public static String addDefaultsToTaskLaunchCmdOpts(String vOpts, Configuration conf, - JavaOptsChecker javaOptsChecker) throws TezException { + JavaOptsChecker javaOptsChecker) throws TezException { String vConfigOpts = ""; String taskDefaultOpts = conf.get(TezConfiguration.TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS, TezConfiguration.TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT); @@ -816,7 +812,7 @@ public static String addDefaultsToTaskLaunchCmdOpts(String vOpts, Configuration @Private @VisibleForTesting public static void addLog4jSystemProperties(String logLevel, - List vargs) { + List vargs) { vargs.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator"); vargs.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE); @@ -838,13 +834,13 @@ public static void addLog4jSystemProperties(String logLevel, } static ConfigurationProto createFinalConfProtoForApp(Configuration amConf, - ServicePluginsDescriptor servicePluginsDescriptor) { + ServicePluginsDescriptor servicePluginsDescriptor) { assert amConf != null; ConfigurationProto.Builder builder = ConfigurationProto.newBuilder(); for (Entry entry : amConf) { String key = entry.getKey(); String val = amConf.get(key); - if(val != null) { + if (val != null) { PlanKeyValuePair.Builder kvp = PlanKeyValuePair.newBuilder(); kvp.setKey(key); kvp.setValue(val); @@ -861,8 +857,6 @@ static ConfigurationProto createFinalConfProtoForApp(Configuration amConf, return builder.build(); } - - /** * Helper function to create a YARN LocalResource * @param fs FileSystem object @@ -872,8 +866,8 @@ static ConfigurationProto createFinalConfProtoForApp(Configuration amConf, * @throws IOException */ static LocalResource createLocalResource(FileSystem fs, Path p, - LocalResourceType type, - LocalResourceVisibility visibility) throws IOException { + LocalResourceType type, + LocalResourceVisibility visibility) throws IOException { LocalResource rsrc = Records.newRecord(LocalResource.class); FileStatus rsrcStat = fs.getFileStatus(p); rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs.resolvePath(rsrcStat @@ -886,7 +880,7 @@ static LocalResource createLocalResource(FileSystem fs, Path p, } private static Path localizeDagPlanAsText(DAGPlan dagPB, FileSystem fs, AMConfiguration amConfig, - String strAppId, Path tezSysStagingPath) throws IOException { + String strAppId, Path tezSysStagingPath) throws IOException { Path textPath = TezCommonUtils.getTezTextPlanStagingPath(tezSysStagingPath, strAppId, dagPB.getName()); FSDataOutputStream dagPBOutTextStream = null; @@ -916,18 +910,18 @@ private static Path localizeDagPlanAsText(DAGPlan dagPB, FileSystem fs, AMConfig } static DAGClientAMProtocolBlockingPB getAMProxy(FrameworkClient frameworkClient, - Configuration conf, ApplicationId applicationId, UserGroupInformation ugi) + Configuration conf, ApplicationId applicationId, UserGroupInformation ugi) throws TezException, IOException { ApplicationReport appReport; try { appReport = frameworkClient.getApplicationReport(applicationId); - if(appReport == null) { + if (appReport == null) { throw new TezUncheckedException("Could not retrieve application report" + " from YARN, applicationId=" + applicationId); } YarnApplicationState appState = appReport.getYarnApplicationState(); - if(appState != YarnApplicationState.RUNNING) { + if (appState != YarnApplicationState.RUNNING) { if (appState == YarnApplicationState.FINISHED || appState == YarnApplicationState.KILLED || appState == YarnApplicationState.FAILED) { @@ -939,7 +933,7 @@ static DAGClientAMProtocolBlockingPB getAMProxy(FrameworkClient frameworkClient, + ", trackingUrl=" + appReport.getTrackingUrl() + ", diagnostics=" + (appReport.getDiagnostics() != null ? appReport.getDiagnostics() - : TezClient.NO_CLUSTER_DIAGNOSTICS_MSG); + : TezClient.NO_CLUSTER_DIAGNOSTICS_MSG); LOG.info(msg); throw new SessionNotRunning(msg); } @@ -957,8 +951,8 @@ static DAGClientAMProtocolBlockingPB getAMProxy(FrameworkClient frameworkClient, @Private public static DAGClientAMProtocolBlockingPB getAMProxy(final Configuration conf, String amHost, - int amRpcPort, org.apache.hadoop.yarn.api.records.Token clientToAMToken, - UserGroupInformation userUgi) throws IOException { + int amRpcPort, org.apache.hadoop.yarn.api.records.Token clientToAMToken, + UserGroupInformation userUgi) throws IOException { final InetSocketAddress serviceAddr = NetUtils.createSocketAddrForHost(amHost, amRpcPort); if (clientToAMToken != null) { @@ -984,8 +978,8 @@ public DAGClientAMProtocolBlockingPB run() throws IOException { } static void createSessionToken(String tokenIdentifier, - JobTokenSecretManager jobTokenSecretManager, - Credentials credentials) { + JobTokenSecretManager jobTokenSecretManager, + Credentials credentials) { JobTokenIdentifier identifier = new JobTokenIdentifier(new Text( tokenIdentifier)); Token sessionToken = new Token(identifier, @@ -1003,9 +997,9 @@ static void createSessionToken(String tokenIdentifier, * @return Modified java opts with computed Xmx value */ public static String maybeAddDefaultMemoryJavaOpts(String javaOpts, Resource resource, - double maxHeapFactor) { + double maxHeapFactor) { if ((javaOpts != null && !javaOpts.isEmpty() - && (javaOpts.contains("-Xmx") || javaOpts.contains("-Xms"))) + && (javaOpts.contains("-Xmx") || javaOpts.contains("-Xms"))) || (resource.getMemory() <= 0)) { return javaOpts; } @@ -1016,14 +1010,14 @@ public static String maybeAddDefaultMemoryJavaOpts(String javaOpts, Resource res if (Double.valueOf("-1") == maxHeapFactor) { maxHeapFactor = resource.getMemory() < TezConstants.TEZ_CONTAINER_SMALL_SLAB_BOUND_MB - ? TezConstants.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_SMALL_SLAB - : TezConstants.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_LARGE_SLAB; + ? TezConstants.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_SMALL_SLAB + : TezConstants.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_LARGE_SLAB; } - int maxMemory = (int)(resource.getMemory() * maxHeapFactor); + int maxMemory = (int) (resource.getMemory() * maxHeapFactor); maxMemory = maxMemory <= 0 ? 1 : maxMemory; return " -Xmx" + maxMemory + "m " - + ( javaOpts != null ? javaOpts : ""); + + (javaOpts != null ? javaOpts : ""); } private static boolean checkAncestorPermissionsForAllUsers(Configuration conf, Path pathComponent, @@ -1075,7 +1069,7 @@ static String constructAMLaunchOpts(TezConfiguration tezConf, Resource capabilit } private static void addVersionInfoToEnv(Map environment, - TezApiVersionInfo versionInfo) { + TezApiVersionInfo versionInfo) { if (!versionInfo.getVersion().equals(VersionInfo.UNKNOWN)) { TezYARNUtils.replaceInEnv(environment, TezConstants.TEZ_CLIENT_VERSION_ENV, versionInfo.getVersion()); @@ -1107,7 +1101,7 @@ public static String[] parseLogParams(String logConfig) { @VisibleForTesting // this method will set the app priority only if the priority config has been defined public static void setApplicationPriority(ApplicationSubmissionContext context, - AMConfiguration amConfig) { + AMConfiguration amConfig) { if (amConfig.getTezConfiguration().get(TezConfiguration.TEZ_AM_APPLICATION_PRIORITY) != null) { // since we already checked not null condition, we are guaranteed that default value // (0 in this case for getInt) will not be returned/used. @@ -1122,7 +1116,6 @@ public static void setApplicationPriority(ApplicationSubmissionContext context, } } - public static byte[] getLocalSha(Path path, Configuration conf) throws IOException { InputStream is = null; try { diff --git a/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java b/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java index f89664d863..fc2e278321 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezYarnClient.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -74,9 +74,9 @@ public YarnClientApplication createApplication() throws YarnException, IOExcepti @Override public ApplicationId submitApplication(ApplicationSubmissionContext appSubmissionContext) throws YarnException, IOException, TezException { - ApplicationId appId= yarnClient.submitApplication(appSubmissionContext); + ApplicationId appId = yarnClient.submitApplication(appSubmissionContext); ApplicationReport appReport = getApplicationReport(appId); - if (appReport.getYarnApplicationState() == YarnApplicationState.FAILED){ + if (appReport.getYarnApplicationState() == YarnApplicationState.FAILED) { throw new TezException("Failed to submit application to YARN" + ", applicationId=" + appId + ", diagnostics=" + appReport.getDiagnostics()); diff --git a/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java b/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java index 47d536fdd9..e2b15b604c 100644 --- a/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java +++ b/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -111,7 +111,6 @@ public class ATSConstants { public static final String COUNTER_MAX_VALUE = "counterMaxValue"; public static final String COUNTER_INSTANCE_COUNT = "counterInstanceCount"; - /* Url related */ public static final String RESOURCE_URI_BASE = "/ws/v1/timeline"; public static final String TEZ_DAG_ID = "TEZ_DAG_ID"; @@ -137,5 +136,4 @@ public class ATSConstants { public static final String CALLER_ID = "callerId"; public static final String CALLER_TYPE = "callerType"; public static final String DESCRIPTION = "description"; - } diff --git a/tez-api/src/main/java/org/apache/tez/common/CachedEntity.java b/tez-api/src/main/java/org/apache/tez/common/CachedEntity.java index 47ccd77e48..ac990f60dd 100644 --- a/tez-api/src/main/java/org/apache/tez/common/CachedEntity.java +++ b/tez-api/src/main/java/org/apache/tez/common/CachedEntity.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-api/src/main/java/org/apache/tez/common/ContainerSignatureMatcher.java b/tez-api/src/main/java/org/apache/tez/common/ContainerSignatureMatcher.java index c0a1245aec..af23b6363a 100644 --- a/tez-api/src/main/java/org/apache/tez/common/ContainerSignatureMatcher.java +++ b/tez-api/src/main/java/org/apache/tez/common/ContainerSignatureMatcher.java @@ -30,35 +30,35 @@ public interface ContainerSignatureMatcher { * Checks the compatibility between the specified container signatures. * * @return true if the first signature is a super set of the second - * signature. + * signature. */ public boolean isSuperSet(Object cs1, Object cs2); - + /** * Checks if the container signatures match exactly + * * @return true if exact match */ public boolean isExactMatch(Object cs1, Object cs2); - + /** * Gets additional resources specified in lr2, which are not present for lr1 - * + * * @param lr1 * @param lr2 * @return additional resources specified in lr2, which are not present for lr1 */ public Map getAdditionalResources(Map lr1, - Map lr2); - + Map lr2); /** * Do a union of 2 signatures * Pre-condition. This function should only be invoked iff cs1 is compatible with cs2. * i.e. isSuperSet should not return false. + * * @param cs1 Signature 1 Original signature * @param cs2 Signature 2 New signature * @return Union of 2 signatures */ public Object union(Object cs1, Object cs2); - } diff --git a/tez-api/src/main/java/org/apache/tez/common/GuavaShim.java b/tez-api/src/main/java/org/apache/tez/common/GuavaShim.java index d9b8796d61..33b59610ce 100644 --- a/tez-api/src/main/java/org/apache/tez/common/GuavaShim.java +++ b/tez-api/src/main/java/org/apache/tez/common/GuavaShim.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -51,4 +51,4 @@ public static Executor directExecutor() { throw new RuntimeException(e); } } -} \ No newline at end of file +} diff --git a/tez-api/src/main/java/org/apache/tez/common/JavaOptsChecker.java b/tez-api/src/main/java/org/apache/tez/common/JavaOptsChecker.java index 3e93446149..f2d64e49e3 100644 --- a/tez-api/src/main/java/org/apache/tez/common/JavaOptsChecker.java +++ b/tez-api/src/main/java/org/apache/tez/common/JavaOptsChecker.java @@ -51,7 +51,7 @@ public void checkOpts(String opts) throws TezException { continue; } - int val = ( matcher.group(2).equals("+") ? 1 : -1 ); + int val = (matcher.group(2).equals("+") ? 1 : -1); if (gcOpts.contains(opt)) { val += 1; } @@ -68,7 +68,7 @@ public void checkOpts(String opts) throws TezException { // which can be specified together. if (gcOpts.size() == 2) { if (gcOpts.contains("UseParNewGC") - && gcOpts.contains("UseConcMarkSweepGC")) { + && gcOpts.contains("UseConcMarkSweepGC")) { return; } } @@ -78,7 +78,5 @@ public void checkOpts(String opts) throws TezException { throw new TezException("Invalid/conflicting GC options found," + " cmdOpts=\"" + opts + "\""); } - } - } diff --git a/tez-api/src/main/java/org/apache/tez/common/Preconditions.java b/tez-api/src/main/java/org/apache/tez/common/Preconditions.java index b32d951101..e7922631a6 100644 --- a/tez-api/src/main/java/org/apache/tez/common/Preconditions.java +++ b/tez-api/src/main/java/org/apache/tez/common/Preconditions.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -68,7 +68,7 @@ private static String format(@Nullable String template, @Nullable Object... args template = String.valueOf(template); // null -> "null" if (args == null) { - args = new Object[] { "(Object[])null" }; + args = new Object[]{"(Object[])null"}; } else { for (int i = 0; i < args.length; i++) { args[i] = lenientToString(args[i]); diff --git a/tez-api/src/main/java/org/apache/tez/common/ProgressHelper.java b/tez-api/src/main/java/org/apache/tez/common/ProgressHelper.java index 289847a96a..ec6e7e947f 100644 --- a/tez-api/src/main/java/org/apache/tez/common/ProgressHelper.java +++ b/tez-api/src/main/java/org/apache/tez/common/ProgressHelper.java @@ -19,14 +19,17 @@ package org.apache.tez.common; import com.google.common.util.concurrent.ThreadFactoryBuilder; + import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicReference; + import org.apache.tez.runtime.api.AbstractLogicalInput; import org.apache.tez.runtime.api.LogicalInput; import org.apache.tez.runtime.api.ProcessorContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -54,7 +57,7 @@ public static final boolean isProgressWithinRange(float val) { } public ProgressHelper(Map inputsParam, - ProcessorContext context, String processorName) { + ProcessorContext context, String processorName) { this.periodicMonitorTaskRef = new AtomicReference<>(null); this.inputs = inputsParam; this.processorContext = context; diff --git a/tez-api/src/main/java/org/apache/tez/common/RPCUtil.java b/tez-api/src/main/java/org/apache/tez/common/RPCUtil.java index caeb822bc7..b092cfd176 100644 --- a/tez-api/src/main/java/org/apache/tez/common/RPCUtil.java +++ b/tez-api/src/main/java/org/apache/tez/common/RPCUtil.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.common; @@ -90,7 +90,6 @@ private static T instantiateSessionNotRunningExcep return instantiateException(cls, re); } - /** * Utility method that unwraps and returns appropriate exceptions. * @@ -123,8 +122,8 @@ public static Void unwrapAndThrowException(ServiceException se) throw instantiateTezException( realClass.asSubclass(SessionNotRunning.class), re); } else if (DAGNotRunningException.class.isAssignableFrom(realClass)) { - throw instantiateTezException( - realClass.asSubclass(DAGNotRunningException.class), re); + throw instantiateTezException( + realClass.asSubclass(DAGNotRunningException.class), re); } else if (TezException.class.isAssignableFrom(realClass)) { throw instantiateTezException( realClass.asSubclass(TezException.class), re); @@ -169,5 +168,4 @@ public static Void unwrapAndThrowNonIOException(ServiceException se) throw new TezException(ioe); } } - } diff --git a/tez-api/src/main/java/org/apache/tez/common/ReflectionUtils.java b/tez-api/src/main/java/org/apache/tez/common/ReflectionUtils.java index 73becdaa6a..a99e7bde8e 100644 --- a/tez-api/src/main/java/org/apache/tez/common/ReflectionUtils.java +++ b/tez-api/src/main/java/org/apache/tez/common/ReflectionUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -58,7 +58,7 @@ private static T getNewInstance(Class clazz) throws TezReflectionExceptio } private static T getNewInstance(Class clazz, Class[] parameterTypes, Object[] parameters) - throws TezReflectionException { + throws TezReflectionException { T instance; try { Constructor constructor = clazz.getConstructor(parameterTypes); @@ -80,7 +80,7 @@ public static T createClazzInstance(String className) throws TezReflectionEx @Private public static T createClazzInstance(String className, Class[] parameterTypes, Object[] parameters) - throws TezReflectionException { + throws TezReflectionException { Class clazz = getClazz(className); @SuppressWarnings("unchecked") T instance = (T) getNewInstance(clazz, parameterTypes, parameters); diff --git a/tez-api/src/main/java/org/apache/tez/common/ServicePluginLifecycle.java b/tez-api/src/main/java/org/apache/tez/common/ServicePluginLifecycle.java index b52b08c58f..404909660f 100644 --- a/tez-api/src/main/java/org/apache/tez/common/ServicePluginLifecycle.java +++ b/tez-api/src/main/java/org/apache/tez/common/ServicePluginLifecycle.java @@ -44,5 +44,4 @@ public interface ServicePluginLifecycle { * Shutdown the service. This will be invoked when the service is shutting down. */ void shutdown() throws Exception; - } diff --git a/tez-api/src/main/java/org/apache/tez/common/TezClassLoader.java b/tez-api/src/main/java/org/apache/tez/common/TezClassLoader.java index 53c9e51fc6..da1ffcbe2f 100644 --- a/tez-api/src/main/java/org/apache/tez/common/TezClassLoader.java +++ b/tez-api/src/main/java/org/apache/tez/common/TezClassLoader.java @@ -24,7 +24,7 @@ /** * ClassLoader to allow addition of new paths to classpath in the runtime. - * + *

* It uses URLClassLoader with this class' classloader as parent classloader. * And hence first delegates the resource loading to parent and then to the URLs * added. The process must be setup to use by invoking setupTezClassLoader() which sets @@ -45,7 +45,7 @@ public TezClassLoader run() { } private TezClassLoader() { - super(new URL[] {}, TezClassLoader.class.getClassLoader()); + super(new URL[]{}, TezClassLoader.class.getClassLoader()); LOG.info( "Created TezClassLoader with parent classloader: {}, thread: {}, system classloader: {}", diff --git a/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java index 5c2876444c..2c1f0fffa2 100644 --- a/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java +++ b/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -73,7 +73,7 @@ public class TezCommonUtils { * function makes sure if the staging directory exists. If not, it creates the * directory with permission TEZ_AM_DIR_PERMISSION. *

- * + * * @param conf * TEZ configuration * @return Fully qualified staging directory @@ -103,7 +103,7 @@ public static Path getTezBaseStagingPath(Configuration conf) { * sub-directory (TEZ_SYSTEM_SUB_DIR/) under the base * staging directory, often provided by user. *

- * + * * @param conf * Tez configuration * @param strAppId @@ -134,7 +134,7 @@ public static Path createTezSystemStagingPath(Configuration conf, String strAppI * its temporary files under this sub-directory. The function normally doesn't * creates any sub-directory under the base staging directory. *

- * + * * @param conf * Tez configuration * @param strAppId @@ -154,7 +154,7 @@ public static Path getTezSystemStagingPath(Configuration conf, String strAppId) *

* Returns a path to store binary configuration *

- * + * * @param tezSysStagingPath * TEZ system level staging directory used for Tez internals * @return path to configuration @@ -168,7 +168,7 @@ public static Path getTezConfStagingPath(Path tezSysStagingPath) { *

* Returns a path to store local resources/session jars *

- * + * * @param tezSysStagingPath * TEZ system level staging directory used for Tez internals * @return path to store the session jars @@ -182,7 +182,7 @@ public static Path getTezAMJarStagingPath(Path tezSysStagingPath) { *

* Returns a path to store binary plan *

- * + * * @param tezSysStagingPath * TEZ system level staging directory used for Tez internals * @return path to store the plan in binary @@ -196,7 +196,7 @@ public static Path getTezBinPlanStagingPath(Path tezSysStagingPath) { *

* Returns a path to store text plan *

- * + * * @param tezSysStagingPath * TEZ system level staging directory used for Tez internals * @param strAppId @@ -207,7 +207,7 @@ public static Path getTezBinPlanStagingPath(Path tezSysStagingPath) { */ @Private public static Path getTezTextPlanStagingPath(Path tezSysStagingPath, String strAppId, - String dagPBName) { + String dagPBName) { String fileName = strAppId + "-" + dagPBName + "-" + TezConstants.TEZ_PB_PLAN_TEXT_NAME; return new Path(tezSysStagingPath, fileName); } @@ -216,7 +216,7 @@ public static Path getTezTextPlanStagingPath(Path tezSysStagingPath, String strA *

* Returns a path to store recovery information *

- * + * * @param tezSysStagingPath * TEZ system level staging directory used for Tez internals * @param conf @@ -237,7 +237,7 @@ public static Path getRecoveryPath(Path tezSysStagingPath, Configuration conf) *

* Returns a path to store app attempt specific recovery details *

- * + * * @param recoveryPath * TEZ recovery directory used for Tez internals * @param attemptID @@ -253,7 +253,7 @@ public static Path getAttemptRecoveryPath(Path recoveryPath, int attemptID) { *

* Returns a path to store DAG specific recovery info *

- * + * * @param attemptRecoverPath * :TEZ system level staging directory used for Tez internals * @param dagID @@ -269,7 +269,7 @@ public static Path getDAGRecoveryPath(Path attemptRecoverPath, String dagID) { *

* Returns a path to store summary info for recovery *

- * + * * @param attemptRecoverPath * TEZ system level staging directory used for Tez internals * @return Summary event path used in recovery @@ -283,7 +283,7 @@ public static Path getSummaryRecoveryPath(Path attemptRecoverPath) { *

* Create a directory with predefined directory permission *

- * + * * @param fs * Filesystem * @param dir @@ -295,8 +295,8 @@ public static void mkDirForAM(FileSystem fs, Path dir) throws IOException { fs.mkdirs(dir, perm); if (!fs.getFileStatus(dir).getPermission().equals(perm)) { LOG.warn("Directory " + dir.toString() + " created with unexpected permissions : " - + fs.getFileStatus(dir).getPermission() + ". Fixing permissions to correct value : " - + perm.toString()); + + fs.getFileStatus(dir).getPermission() + ". Fixing permissions to correct value : " + + perm.toString()); fs.setPermission(dir, perm); } } @@ -306,7 +306,7 @@ public static void mkDirForAM(FileSystem fs, Path dir) throws IOException { * Create a file with TEZ_AM_FILE_PERMISSION permission and * returns OutputStream *

- * + * * @param fs * Filesystem * @param filePath @@ -317,9 +317,9 @@ public static void mkDirForAM(FileSystem fs, Path dir) throws IOException { public static FSDataOutputStream createFileForAM(FileSystem fs, Path filePath) throws IOException { return FileSystem.create(fs, filePath, new FsPermission(TEZ_AM_FILE_PERMISSION)); } - + public static void addAdditionalLocalResources(Map additionalLrs, - Map originalLRs, String logContext) { + Map originalLRs, String logContext) { // TODO TEZ-1798. Handle contents of Tez archives for duplicate LocalResource checks if (additionalLrs != null && !additionalLrs.isEmpty()) { StringBuilder sb = new StringBuilder(); @@ -503,7 +503,7 @@ public static ByteBuffer serializeServiceData(Token jobToken } public static String getSystemPropertiesToLog(Configuration conf) { - Collection keys = conf.getTrimmedStringCollection( + Collection keys = conf.getTrimmedStringCollection( TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG); if (keys.isEmpty()) { keys = TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG_DEFAULT; @@ -556,7 +556,7 @@ public static long getAMClientHeartBeatPollIntervalMillis(Configuration conf, pollInterval); } return Math.max(TezConstants.TEZ_AM_CLIENT_HEARTBEAT_POLL_INTERVAL_MILLIS_MINIMUM, - heartbeatIntervalMillis/buckets); + heartbeatIntervalMillis / buckets); } public static long getDAGSessionTimeout(Configuration conf) { @@ -576,7 +576,7 @@ public static long getDAGSessionTimeout(Configuration conf) { public static int getJavaVersion() { String javaVersionString = System.getProperty("java.version"); return javaVersionString.split("\\.")[0].equals("1") - ? Integer.parseInt(javaVersionString.split("\\.")[1]) // "1.8" -> 8 - : Integer.parseInt(javaVersionString.split("\\.")[0]); // "9.x" -> 9, "11.x" -> 11 + ? Integer.parseInt(javaVersionString.split("\\.")[1]) // "1.8" -> 8 + : Integer.parseInt(javaVersionString.split("\\.")[0]); // "9.x" -> 9, "11.x" -> 11 } } diff --git a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java index 1c0be98dcc..d539f579e1 100644 --- a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java +++ b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java @@ -55,7 +55,7 @@ public class TezUtils { /** * Allows changing the log level for task / AM logging.

- * + *

* Adds the JVM system properties necessary to configure * {@link org.apache.hadoop.yarn.ContainerLogAppender}. * @@ -70,8 +70,7 @@ public static void addLog4jSystemProperties(String logLevel, /** * Convert a Configuration to compressed ByteString using Protocol buffer * - * @param conf - * : Configuration to be converted + * @param conf : Configuration to be converted * @return PB ByteString (compressed) * @throws java.io.IOException */ @@ -92,7 +91,6 @@ public static ByteString createByteStringFromConf(Configuration conf) throws IOE /** * Convert a Configuration to a {@link org.apache.tez.dag.api.UserPayload}

* - * * @param conf configuration to be converted * @return an instance of {@link org.apache.tez.dag.api.UserPayload} * @throws java.io.IOException @@ -117,7 +115,7 @@ private static DAGProtos.ConfigurationProto createConfProto(SnappyInputStream un */ public static Configuration createConfFromByteString(ByteString byteString) throws IOException { Objects.requireNonNull(byteString, "ByteString must be specified"); - try(SnappyInputStream uncompressIs = new SnappyInputStream(byteString.newInput());) { + try (SnappyInputStream uncompressIs = new SnappyInputStream(byteString.newInput());) { DAGProtos.ConfigurationProto confProto = createConfProto(uncompressIs); Configuration conf = new Configuration(false); readConfFromPB(confProto, conf); @@ -132,7 +130,7 @@ public static Configuration createConfFromBaseConfAndPayload(TaskContext context Configuration configuration = new Configuration(baseConf); UserPayload payload = context.getUserPayload(); ByteString byteString = ByteString.copyFrom(payload.getPayload()); - try(SnappyInputStream uncompressIs = new SnappyInputStream(byteString.newInput())) { + try (SnappyInputStream uncompressIs = new SnappyInputStream(byteString.newInput())) { DAGProtos.ConfigurationProto confProto = createConfProto(uncompressIs); readConfFromPB(confProto, configuration); TezClassLoader.setupForConfiguration(configuration); @@ -142,7 +140,7 @@ public static Configuration createConfFromBaseConfAndPayload(TaskContext context public static void addToConfFromByteString(Configuration configuration, ByteString byteString) throws IOException { - try(SnappyInputStream uncompressIs = new SnappyInputStream(byteString.newInput())) { + try (SnappyInputStream uncompressIs = new SnappyInputStream(byteString.newInput())) { DAGProtos.ConfigurationProto confProto = createConfProto(uncompressIs); readConfFromPB(confProto, configuration); TezClassLoader.setupForConfiguration(configuration); @@ -162,7 +160,6 @@ public static Configuration createConfFromUserPayload(UserPayload payload) throw return createConfFromByteString(ByteString.copyFrom(payload.getPayload())); } - private static void writeConfInPB(OutputStream dos, Configuration conf) throws IOException { DAGProtos.ConfigurationProto.Builder confProtoBuilder = DAGProtos.ConfigurationProto.newBuilder(); populateConfProtoFromEntries(conf, confProtoBuilder); @@ -191,7 +188,7 @@ public static String convertToHistoryText(String description, Configuration conf Entry entry = iter.next(); String key = entry.getKey(); String val = conf.get(entry.getKey()); - if(val != null) { + if (val != null) { confJson.put(key, val); } else { LOG.debug("null value in Configuration after replacement for key={}. Skipping.", key); @@ -209,14 +206,13 @@ public static String convertToHistoryText(Configuration conf) { return convertToHistoryText(null, conf); } - /* Copy each Map.Entry with non-null value to DAGProtos.ConfigurationProto */ public static void populateConfProtoFromEntries(Iterable> params, - DAGProtos.ConfigurationProto.Builder confBuilder) { - for(Map.Entry entry : params) { + DAGProtos.ConfigurationProto.Builder confBuilder) { + for (Map.Entry entry : params) { String key = entry.getKey(); String val = entry.getValue(); - if(val != null) { + if (val != null) { DAGProtos.PlanKeyValuePair.Builder kvp = DAGProtos.PlanKeyValuePair.newBuilder(); kvp.setKey(key); kvp.setValue(val); @@ -226,5 +222,4 @@ public static void populateConfProtoFromEntries(Iterable * http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -41,33 +41,33 @@ public class TezYARNUtils { public static final String ENV_NAME_REGEX = "[A-Za-z_][A-Za-z0-9_]*"; private static final Pattern VAR_SUBBER = - Pattern.compile(Shell.getEnvironmentVariableRegex()); + Pattern.compile(Shell.getEnvironmentVariableRegex()); private static final Pattern VARVAL_SPLITTER = Pattern.compile( - "(?<=^|,)" // preceded by ',' or line begin - + '(' + ENV_NAME_REGEX + ')' // var group - + '=' - + "([^,]*)" // val group + "(?<=^|,)" // preceded by ',' or line begin + + '(' + ENV_NAME_REGEX + ')' // var group + + '=' + + "([^,]*)" // val group ); public static String getFrameworkClasspath(Configuration conf, boolean usingArchive) { StringBuilder classpathBuilder = new StringBuilder(); boolean userClassesTakesPrecedence = - conf.getBoolean(TezConfiguration.TEZ_USER_CLASSPATH_FIRST, - TezConfiguration.TEZ_USER_CLASSPATH_FIRST_DEFAULT); + conf.getBoolean(TezConfiguration.TEZ_USER_CLASSPATH_FIRST, + TezConfiguration.TEZ_USER_CLASSPATH_FIRST_DEFAULT); if (userClassesTakesPrecedence) { addUserSpecifiedClasspath(classpathBuilder, conf); } - String [] tezLibUrisClassPath = conf.getStrings(TezConfiguration.TEZ_LIB_URIS_CLASSPATH); + String[] tezLibUrisClassPath = conf.getStrings(TezConfiguration.TEZ_LIB_URIS_CLASSPATH); - if(!conf.getBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, false) && - tezLibUrisClassPath != null && tezLibUrisClassPath.length != 0) { - for(String c : tezLibUrisClassPath) { + if (!conf.getBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, false) && + tezLibUrisClassPath != null && tezLibUrisClassPath.length != 0) { + for (String c : tezLibUrisClassPath) { classpathBuilder.append(c.trim()) - .append(File.pathSeparator); + .append(File.pathSeparator); } } else { - if(conf.getBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, false)) { + if (conf.getBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, false)) { LOG.info("Ignoring '" + TezConfiguration.TEZ_LIB_URIS + "' since '" + TezConfiguration.TEZ_IGNORE_LIB_URIS + "' is set to true "); } @@ -116,7 +116,7 @@ public static String getFrameworkClasspath(Configuration conf, boolean usingArch } private static void addUserSpecifiedClasspath(StringBuilder classpathBuilder, - Configuration conf) { + Configuration conf) { // Add any additional user-specified classpath String additionalClasspath = conf.get(TezConfiguration.TEZ_CLUSTER_ADDITIONAL_CLASSPATH_PREFIX); if (additionalClasspath != null && !additionalClasspath.trim().isEmpty()) { @@ -132,7 +132,7 @@ private static void addUserSpecifiedClasspath(StringBuilder classpathBuilder, } public static void appendToEnvFromInputString(Map env, - String envString, String classPathSeparator) { + String envString, String classPathSeparator) { if (envString != null && envString.length() > 0) { Matcher varValMatcher = VARVAL_SPLITTER.matcher(envString); while (varValMatcher.find()) { @@ -157,14 +157,14 @@ public static void appendToEnvFromInputString(Map env, } } } - + public static void setEnvIfAbsentFromInputString(Map env, - String envString) { + String envString) { if (envString != null && envString.length() > 0) { String childEnvs[] = envString.split(","); for (String cEnv : childEnvs) { String[] parts = cEnv.split("="); // split on '=' - Matcher m = VAR_SUBBER .matcher(parts[1]); + Matcher m = VAR_SUBBER.matcher(parts[1]); StringBuffer sb = new StringBuffer(); while (m.find()) { String var = m.group(1); @@ -184,7 +184,7 @@ public static void setEnvIfAbsentFromInputString(Map env, } } } - + public static void addToEnvironment( Map environment, String variable, String value, String classPathSeparator) { @@ -194,7 +194,7 @@ public static void addToEnvironment( } else { val = val + classPathSeparator + value; } - environment.put(StringInterner.weakIntern(variable), + environment.put(StringInterner.weakIntern(variable), StringInterner.weakIntern(val)); } @@ -204,8 +204,8 @@ private static void putIfAbsent(Map env, String key, String valu } } - public static void setupDefaultEnv(Map env, Configuration conf, String userEnvKey, String userEnvDefault, - String clusterDefaultEnvKey, String clusterDefaultEnvDefault, boolean usingArchive) { + public static void setupDefaultEnv(Map env, Configuration conf, String userEnvKey, String userEnvDefault, + String clusterDefaultEnvKey, String clusterDefaultEnvDefault, boolean usingArchive) { // Setup the CLASSPATH in environment // i.e. add { Hadoop jars, job jar, CWD } to classpath. String classpath = getFrameworkClasspath(conf, usingArchive); @@ -228,5 +228,4 @@ public static void setupDefaultEnv(Map env, Configuration conf, public static void replaceInEnv(Map env, String key, String value) { env.put(StringInterner.weakIntern(key), StringInterner.weakIntern(value)); } - } diff --git a/tez-api/src/main/java/org/apache/tez/common/VersionInfo.java b/tez-api/src/main/java/org/apache/tez/common/VersionInfo.java index 1afde0d3f4..de5bcf3fe7 100644 --- a/tez-api/src/main/java/org/apache/tez/common/VersionInfo.java +++ b/tez-api/src/main/java/org/apache/tez/common/VersionInfo.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -69,7 +69,7 @@ protected VersionInfo(String component) { @VisibleForTesting @Private protected VersionInfo(String component, String version, String revision, String buildTime, - String scmUrl) { + String scmUrl) { this.info = new Properties(); this.component = component; info.setProperty(VERSION, version); @@ -127,5 +127,4 @@ public static void main(String[] args) { System.out.println("This command was run using " + ClassUtil.findContainingJar(VersionInfo.class)); } - } diff --git a/tez-api/src/main/java/org/apache/tez/common/annotation/ConfigurationClass.java b/tez-api/src/main/java/org/apache/tez/common/annotation/ConfigurationClass.java index 6b8fdaf178..6cafcd038b 100644 --- a/tez-api/src/main/java/org/apache/tez/common/annotation/ConfigurationClass.java +++ b/tez-api/src/main/java/org/apache/tez/common/annotation/ConfigurationClass.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-api/src/main/java/org/apache/tez/common/annotation/ConfigurationProperty.java b/tez-api/src/main/java/org/apache/tez/common/annotation/ConfigurationProperty.java index f7d37e20e4..f72a35075c 100644 --- a/tez-api/src/main/java/org/apache/tez/common/annotation/ConfigurationProperty.java +++ b/tez-api/src/main/java/org/apache/tez/common/annotation/ConfigurationProperty.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,6 +31,8 @@ public @interface ConfigurationProperty { public boolean mandatory() default false; + public String type() default "string"; + public String[] validValues() default {}; } diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounter.java b/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounter.java index befc7c023e..4076832b88 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounter.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounter.java @@ -38,8 +38,8 @@ public synchronized boolean equals(Object genericRight) { synchronized (genericRight) { TezCounter right = (TezCounter) genericRight; return getName().equals(right.getName()) && - getDisplayName().equals(right.getDisplayName()) && - getValue() == right.getValue(); + getDisplayName().equals(right.getDisplayName()) && + getValue() == right.getValue(); } } return false; diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounterGroup.java b/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounterGroup.java index 1d1b56d062..4af3692857 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounterGroup.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounterGroup.java @@ -128,9 +128,10 @@ public T findCounter(String counterName) { /** * Abstract factory method to create a new counter of type T + * * @param counterName of the counter * @param displayName of the counter - * @param value of the counter + * @param value of the counter * @return a new counter */ protected abstract T newCounter(String counterName, String displayName, @@ -138,6 +139,7 @@ protected abstract T newCounter(String counterName, String displayName, /** * Abstract factory method to create a new counter of type T + * * @return a new counter object */ protected abstract T newCounter(); @@ -154,7 +156,7 @@ public Iterator iterator() { public synchronized void write(DataOutput out) throws IOException { Text.writeString(out, displayName); WritableUtils.writeVInt(out, counters.size()); - for(TezCounter counter: counters.values()) { + for (TezCounter counter : counters.values()) { counter.write(out); } } diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounters.java b/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounters.java index 5910164318..d898747501 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounters.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounters.java @@ -52,7 +52,7 @@ @Public @Unstable public abstract class AbstractCounters> + G extends CounterGroupBase> implements Writable, Iterable { protected static final Logger LOG = LoggerFactory.getLogger("mapreduce.Counters"); @@ -68,17 +68,20 @@ public abstract class AbstractCounters groupFactory; // For framework counter serialization without strings - enum GroupType { FRAMEWORK, FILESYSTEM }; + enum GroupType {FRAMEWORK, FILESYSTEM} + + ; // Writes only framework and fs counters if false. private boolean writeAllCounters = true; private static final Map legacyMap = Maps.newHashMap(); + static { legacyMap.put("org.apache.hadoop.mapred.Task$Counter", - TaskCounter.class.getName()); + TaskCounter.class.getName()); legacyMap.put("org.apache.hadoop.mapred.JobInProgress$Counter", - JobCounter.class.getName()); + JobCounter.class.getName()); legacyMap.put("FileSystemCounters", FileSystemCounter.class.getName()); } @@ -91,9 +94,10 @@ public AbstractCounters(CounterGroupFactory gf) { /** * Construct from another counters object. - * @param type of the other counter - * @param type of the other counter group - * @param counters the counters object to copy + * + * @param type of the other counter + * @param type of the other counter group + * @param counters the counters object to copy * @param groupFactory the factory for new groups */ @InterfaceAudience.Private @@ -101,18 +105,20 @@ public AbstractCounters(CounterGroupFactory gf) { AbstractCounters(AbstractCounters counters, CounterGroupFactory groupFactory) { this.groupFactory = groupFactory; - for(G1 group: counters) { + for (G1 group : counters) { String name = group.getName(); G newGroup = groupFactory.newGroup(name, group.getDisplayName(), limits); (isFrameworkGroup(name) ? fgroups : groups).put(name, newGroup); - for(TezCounter counter: group) { + for (TezCounter counter : group) { newGroup.addCounter(counter.getName(), counter.getDisplayName(), - counter.getValue()); + counter.getValue()); } } } - /** Add a group. + /** + * Add a group. + * * @param group object to add * @return the group */ @@ -130,7 +136,8 @@ public synchronized G addGroup(G group) { /** * Add a new group - * @param name of the group + * + * @param name of the group * @param displayName of the group * @return the group */ @@ -141,7 +148,8 @@ public G addGroup(String name, String displayName) { /** * Find a counter, create one if necessary - * @param groupName of the counter + * + * @param groupName of the counter * @param counterName name of the counter * @return the matching counter */ @@ -153,6 +161,7 @@ public C findCounter(String groupName, String counterName) { /** * Find the counter for the given enum. The same enum will always return the * same counter. + * * @param key the counter key * @return the matching counter object */ @@ -167,8 +176,9 @@ public synchronized C findCounter(Enum key) { /** * Find the file system counter for the given scheme and enum. + * * @param scheme of the file system - * @param key the enum of the counter + * @param key the enum of the counter * @return the file system counter */ @InterfaceAudience.Private @@ -180,14 +190,15 @@ public synchronized C findCounter(String scheme, FileSystemCounter key) { /** * Returns the names of all counter classes. + * * @return Set of counter names. */ public synchronized Iterable getGroupNames() { HashSet deprecated = new HashSet(); - for(Map.Entry entry : legacyMap.entrySet()) { + for (Map.Entry entry : legacyMap.entrySet()) { String newGroup = entry.getValue(); boolean isFGroup = isFrameworkGroup(newGroup); - if(isFGroup ? fgroups.containsKey(newGroup) : groups.containsKey(newGroup)) { + if (isFGroup ? fgroups.containsKey(newGroup) : groups.containsKey(newGroup)) { deprecated.add(entry.getKey()); } } @@ -197,12 +208,13 @@ public synchronized Iterable getGroupNames() { @Override public Iterator iterator() { return Iterators.concat(fgroups.values().iterator(), - groups.values().iterator()); + groups.values().iterator()); } /** * Returns the named counter group, or an empty group if there is none * with the specified name. + * * @param groupName name of the group * @return the group */ @@ -237,6 +249,7 @@ public synchronized G getGroup(String groupName) { /** * Returns the total number of counters, by summing the number of counters * in each group. + * * @return the total number of counters */ public synchronized int countCounters() { @@ -280,8 +293,8 @@ public synchronized void write(DataOutput out) throws IOException { public synchronized void readFields(DataInput in) throws IOException { int version = WritableUtils.readVInt(in); if (version != groupFactory.version()) { - throw new IOException("Counters version mismatch, expected "+ - groupFactory.version() +" got "+ version); + throw new IOException("Counters version mismatch, expected " + + groupFactory.version() + " got " + version); } int numFGroups = WritableUtils.readVInt(in); fgroups.clear(); @@ -297,7 +310,7 @@ public synchronized void readFields(DataInput in) throws IOException { group = groupFactory.newFrameworkGroup(WritableUtils.readVInt(in)); break; default: // Silence dumb compiler, as it would've thrown earlier - throw new IOException("Unexpected counter group type: "+ groupType); + throw new IOException("Unexpected counter group type: " + groupType); } group.readFields(in); fgroups.put(group.getName(), group); @@ -333,16 +346,17 @@ public synchronized String toShortString() { /** * Return textual representation of the counter values. + * * @return the string */ @Override public synchronized String toString() { StringBuilder sb = new StringBuilder("Counters: " + countCounters()); - for (G group: this) { + for (G group : this) { sb.append("\n\t").append(group.getDisplayName()); - for (TezCounter counter: group) { + for (TezCounter counter : group) { sb.append("\n\t\t").append(counter.getDisplayName()).append("=") - .append(counter.getValue()); + .append(counter.getValue()); } } return sb.toString(); @@ -351,6 +365,7 @@ public synchronized String toString() { /** * Increments multiple counters by their amounts in another Counters * instance. + * * @param other the other Counters instance */ public synchronized void incrAllCounters(AbstractCounters other) { @@ -360,10 +375,11 @@ public synchronized void incrAllCounters(AbstractCounters other) { /** * Increments multiple counters by their amounts in another Counters * instance. + * * @param other the other Counters instance */ public synchronized void aggrAllCounters(AbstractCounters other) { - for(G right : other) { + for (G right : other) { String groupName = right.getName(); G left = (isFrameworkGroup(groupName) ? fgroups : groups).get(groupName); if (left == null) { @@ -378,7 +394,7 @@ public synchronized void aggrAllCounters(AbstractCounters other) { public boolean equals(Object genericRight) { if (genericRight instanceof AbstractCounters) { return Iterators.elementsEqual(iterator(), - ((AbstractCounters)genericRight).iterator()); + ((AbstractCounters) genericRight).iterator()); } return false; } @@ -390,9 +406,10 @@ public int hashCode() { /** * Set the "writeAllCounters" option to true or false - * @param send if true all counters would be serialized, otherwise only - * framework counters would be serialized in - * {@link #write(DataOutput)} + * + * @param send if true all counters would be serialized, otherwise only + * framework counters would be serialized in + * {@link #write(DataOutput)} */ @InterfaceAudience.Private public void setWriteAllCounters(boolean send) { @@ -401,6 +418,7 @@ public void setWriteAllCounters(boolean send) { /** * Get the "writeAllCounters" option + * * @return true of all counters would serialized */ @InterfaceAudience.Private diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/AggregateFrameworkCounter.java b/tez-api/src/main/java/org/apache/tez/common/counters/AggregateFrameworkCounter.java index aa7d4462cf..93b59ff013 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/AggregateFrameworkCounter.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/AggregateFrameworkCounter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,8 +21,8 @@ import org.apache.tez.common.counters.FrameworkCounterGroup.FrameworkCounter; @SuppressWarnings("rawtypes") -public class AggregateFrameworkCounter> extends FrameworkCounter implements AggregateTezCounter { - +public class AggregateFrameworkCounter> extends FrameworkCounter implements AggregateTezCounter { + private long min = Long.MAX_VALUE; private long max = Long.MIN_VALUE; private long count = 0; @@ -36,7 +36,7 @@ public AggregateFrameworkCounter(Enum ref, String groupName) { public void increment(long incr) { throw new IllegalArgumentException("Cannot increment an aggregate counter directly"); } - + @Override public void aggregate(TezCounter other) { final long val = other.getValue(); @@ -71,15 +71,14 @@ public long getMin() { public long getMax() { return max; } - + @SuppressWarnings("unchecked") public FrameworkCounter asFrameworkCounter() { - return ((FrameworkCounter)this); + return ((FrameworkCounter) this); } @Override public long getCount() { return count; } - } diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/AggregateTezCounter.java b/tez-api/src/main/java/org/apache/tez/common/counters/AggregateTezCounter.java index bf711dae86..4a250f80f6 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/AggregateTezCounter.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/AggregateTezCounter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,7 +25,6 @@ public interface AggregateTezCounter { public abstract long getMin(); public abstract long getMax(); - - public abstract long getCount(); -} \ No newline at end of file + public abstract long getCount(); +} diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/AggregateTezCounterDelegate.java b/tez-api/src/main/java/org/apache/tez/common/counters/AggregateTezCounterDelegate.java index ae2ca7b0c6..744414f816 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/AggregateTezCounterDelegate.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/AggregateTezCounterDelegate.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -32,7 +32,7 @@ public class AggregateTezCounterDelegate extends AbstractC public AggregateTezCounterDelegate(T child) { this.child = child; } - + @Override public String getName() { return child.getName(); // this is a pass-through @@ -57,7 +57,7 @@ public void setValue(long value) { public void increment(long incr) { throw new UnsupportedOperationException("Cannot increment an aggregate counter"); } - + /* (non-Javadoc) * @see org.apache.tez.common.counters.AggregateTezCounter#aggregate(org.apache.tez.common.counters.TezCounter) */ diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/AggregateTezCounters.java b/tez-api/src/main/java/org/apache/tez/common/counters/AggregateTezCounters.java index 332c24a960..db137246ce 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/AggregateTezCounters.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/AggregateTezCounters.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,13 +19,13 @@ package org.apache.tez.common.counters; public class AggregateTezCounters extends TezCounters { - + private static final GroupFactory groupFactory = new GroupFactory(); - + public AggregateTezCounters() { super(groupFactory); } - + // Mix framework group implementation into CounterGroup interface private static class AggregateFrameworkGroupImpl> extends FrameworkCounterGroup implements CounterGroup { @@ -97,9 +97,10 @@ private static class GroupFactory @Override protected > FrameworkGroupFactory - newFrameworkGroupFactory(final Class cls) { + newFrameworkGroupFactory(final Class cls) { return new FrameworkGroupFactory() { - @Override public CounterGroup newGroup(String name) { + @Override + public CounterGroup newGroup(String name) { return new AggregateFrameworkGroupImpl(cls); // impl in this package } }; diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/CounterGroup.java b/tez-api/src/main/java/org/apache/tez/common/counters/CounterGroup.java index c13e084b6a..88e0e52732 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/CounterGroup.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/CounterGroup.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/CounterGroupBase.java b/tez-api/src/main/java/org/apache/tez/common/counters/CounterGroupBase.java index 216d2f4c75..8bfc4044a0 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/CounterGroupBase.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/CounterGroupBase.java @@ -35,38 +35,45 @@ public interface CounterGroupBase /** * Get the internal name of the group + * * @return the internal name */ String getName(); /** * Get the display name of the group. + * * @return the human readable name */ String getDisplayName(); /** * Set the display name of the group + * * @param displayName of the group */ void setDisplayName(String displayName); - /** Add a counter to this group. + /** + * Add a counter to this group. + * * @param counter to add */ void addCounter(T counter); /** * Add a counter to this group - * @param name of the counter + * + * @param name of the counter * @param displayName of the counter - * @param value of the counter + * @param value of the counter * @return the counter */ T addCounter(String name, String displayName, long value); /** * Find a counter in the group. + * * @param counterName the name of the counter * @param displayName the display name of the counter * @return the counter that was found or added @@ -75,14 +82,16 @@ public interface CounterGroupBase /** * Find a counter in the group + * * @param counterName the name of the counter - * @param create create the counter if not found if true + * @param create create the counter if not found if true * @return the counter that was found or added or null if create is false */ T findCounter(String counterName, boolean create); /** * Find a counter in the group. + * * @param counterName the name of the counter * @return the counter that was found or added */ @@ -95,14 +104,16 @@ public interface CounterGroupBase /** * Increment all counters by a group of counters - * @param rightGroup the group to be added to this group + * + * @param rightGroup the group to be added to this group */ @Deprecated void incrAllCounters(CounterGroupBase rightGroup); - + /** * Aggregate all counters by a group of counters - * @param rightGroup the group to be added to this group + * + * @param rightGroup the group to be added to this group */ public default void aggrAllCounters(CounterGroupBase rightGroup) { incrAllCounters(rightGroup); diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/CounterGroupFactory.java b/tez-api/src/main/java/org/apache/tez/common/counters/CounterGroupFactory.java index ee9b45febd..eeb1d94384 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/CounterGroupFactory.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/CounterGroupFactory.java @@ -35,7 +35,7 @@ */ @InterfaceAudience.Private public abstract class CounterGroupFactory> { + G extends CounterGroupBase> { public interface FrameworkGroupFactory { F newGroup(String name); @@ -48,6 +48,7 @@ public interface FrameworkGroupFactory { private static final String FS_GROUP_NAME = FileSystemCounter.class.getName(); private final Map> fmap = Maps.newHashMap(); + { // Add builtin counter class here and the version when changed. addFrameworkGroup(TaskCounter.class); @@ -73,6 +74,7 @@ private static synchronized void updateFrameworkGroupMapping(Class cls) { /** * Required override to return a new framework group factory + * * @param type of the counter enum class * @param cls the counter enum class * @return a new framework group factory @@ -82,7 +84,8 @@ private static synchronized void updateFrameworkGroupMapping(Class cls) { /** * Create a new counter group - * @param name of the group + * + * @param name of the group * @param limits the counters limits policy object * @return a new counter group */ @@ -92,9 +95,10 @@ public G newGroup(String name, Limits limits) { /** * Create a new counter group - * @param name of the group + * + * @param name of the group * @param displayName of the group - * @param limits the counters limits policy object + * @param limits the counters limits policy object * @return a new counter group */ public G newGroup(String name, String displayName, Limits limits) { @@ -110,12 +114,13 @@ public G newGroup(String name, String displayName, Limits limits) { /** * Create a new framework group + * * @param id of the group * @return a new framework group */ public G newFrameworkGroup(int id) { String name; - synchronized(CounterGroupFactory.class) { + synchronized (CounterGroupFactory.class) { if (id < 0 || id >= i2s.size()) throwBadFrameGroupIdException(id); name = i2s.get(id); // should not throw here. } @@ -126,6 +131,7 @@ public G newFrameworkGroup(int id) { /** * Get the id of a framework group + * * @param name of the group * @return the framework group id */ @@ -146,7 +152,7 @@ public int version() { * Check whether a group name is a name of a framework group (including * the filesystem group). * - * @param name to check + * @param name to check * @return true for framework group names */ public static synchronized boolean isFrameworkGroup(String name) { @@ -154,18 +160,19 @@ public static synchronized boolean isFrameworkGroup(String name) { } private static void throwBadFrameGroupIdException(int id) { - throw new IllegalArgumentException("bad framework group id: "+ id); + throw new IllegalArgumentException("bad framework group id: " + id); } private static void throwBadFrameworkGroupNameException(String name) { - throw new IllegalArgumentException("bad framework group name: "+ name); + throw new IllegalArgumentException("bad framework group name: " + name); } /** * Abstract factory method to create a generic (vs framework) counter group - * @param name of the group + * + * @param name of the group * @param displayName of the group - * @param limits limits of the counters + * @param limits limits of the counters * @return a new generic counter group */ protected abstract G newGenericGroup(String name, String displayName, @@ -173,6 +180,7 @@ protected abstract G newGenericGroup(String name, String displayName, /** * Abstract factory method to create a file system counter group + * * @return a new file system counter group */ protected abstract G newFileSystemGroup(); diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/DAGCounter.java b/tez-api/src/main/java/org/apache/tez/common/counters/DAGCounter.java index 1ef56778c3..2b7a0d162c 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/DAGCounter.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/DAGCounter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,7 +26,7 @@ @InterfaceAudience.Public @InterfaceStability.Evolving public enum DAGCounter { - NUM_FAILED_TASKS, + NUM_FAILED_TASKS, NUM_KILLED_TASKS, NUM_SUCCEEDED_TASKS, TOTAL_LAUNCHED_TASKS, diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/FileSystemCounterGroup.java b/tez-api/src/main/java/org/apache/tez/common/counters/FileSystemCounterGroup.java index 76e5235d27..9b3d115ffd 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/FileSystemCounterGroup.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/FileSystemCounterGroup.java @@ -55,7 +55,7 @@ public abstract class FileSystemCounterGroup // C[] would need Array.newInstance which requires a Class reference. // Just a few local casts probably worth not having to carry it around. private final Map map = - new ConcurrentSkipListMap(); + new ConcurrentSkipListMap(); private String displayName = StringInterner.weakIntern("File System Counters"); private static final Joiner NAME_JOINER = Joiner.on('_'); @@ -133,8 +133,7 @@ public void addCounter(C counter) { if (counter instanceof FileSystemCounterGroup.FSCounter) { FSCounter c = (FSCounter) counter; ours = findCounter(c.scheme, c.key); - } - else { + } else { ours = findCounter(counter.getName()); } ours.setValue(counter.getValue()); @@ -154,7 +153,7 @@ private String[] parseCounterName(String counterName) { throw new IllegalArgumentException("bad fs counter name"); } return new String[]{counterName.substring(0, schemeEnd), - counterName.substring(schemeEnd + 1)}; + counterName.substring(schemeEnd + 1)}; } @Override @@ -167,8 +166,7 @@ public C findCounter(String counterName, boolean create) { try { String[] pair = parseCounterName(counterName); return findCounter(pair[0], FileSystemCounter.valueOf(pair[1])); - } - catch (Exception e) { + } catch (Exception e) { if (create) throw new IllegalArgumentException(e); return null; } @@ -188,8 +186,7 @@ public synchronized C findCounter(String scheme, FileSystemCounter key) { counters = new Object[FileSystemCounter.values().length]; map.put(canonicalScheme, counters); counters[ord] = newCounter(canonicalScheme, key); - } - else if (counters[ord] == null) { + } else if (counters[ord] == null) { counters[ord] = newCounter(canonicalScheme, key); } return (C) counters[ord]; @@ -200,16 +197,17 @@ private String checkScheme(String scheme) { String interned = schemes.putIfAbsent(fixed, fixed); if (schemes.size() > MAX_NUM_SCHEMES) { // mistakes or abuses - throw new IllegalArgumentException("too many schemes? "+ schemes.size() + - " when process scheme: "+ scheme); + throw new IllegalArgumentException("too many schemes? " + schemes.size() + + " when process scheme: " + scheme); } return interned == null ? fixed : interned; } /** * Abstract factory method to create a file system counter + * * @param scheme of the file system - * @param key the enum of the file system counter + * @param key the enum of the file system counter * @return a new file system counter */ protected abstract C newCounter(String scheme, FileSystemCounter key); @@ -233,8 +231,8 @@ public void aggrAllCounters(CounterGroupBase other) { if (Objects.requireNonNull(other.getUnderlyingGroup(), "other group") instanceof FileSystemCounterGroup) { for (TezCounter counter : other) { - FSCounter c = (FSCounter) ((TezCounter)counter).getUnderlyingCounter(); - findCounter(c.scheme, c.key) .aggregate(counter); + FSCounter c = (FSCounter) ((TezCounter) counter).getUnderlyingCounter(); + findCounter(c.scheme, c.key).aggregate(counter); } } } @@ -251,7 +249,7 @@ public void write(DataOutput out) throws IOException { WritableUtils.writeVInt(out, numSetCounters(entry.getValue())); for (Object counter : entry.getValue()) { if (counter == null) continue; - FSCounter c = (FSCounter) ((TezCounter)counter).getUnderlyingCounter(); + FSCounter c = (FSCounter) ((TezCounter) counter).getUnderlyingCounter(); WritableUtils.writeVInt(out, c.key.ordinal()); // key WritableUtils.writeVLong(out, c.getValue()); // value } @@ -284,6 +282,7 @@ public Iterator iterator() { Iterator it = map.values().iterator(); Object[] counters = it.hasNext() ? it.next() : null; int i = 0; + @Override protected C computeNext() { while (counters != null) { diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/FrameworkCounterGroup.java b/tez-api/src/main/java/org/apache/tez/common/counters/FrameworkCounterGroup.java index cb27741283..50a2022742 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/FrameworkCounterGroup.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/FrameworkCounterGroup.java @@ -18,8 +18,6 @@ package org.apache.tez.common.counters; - - import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -154,8 +152,7 @@ public C findCounter(String counterName, String displayName) { public C findCounter(String counterName, boolean create) { try { return findCounter(valueOf(counterName)); - } - catch (Exception e) { + } catch (Exception e) { if (create) throw new IllegalArgumentException(e); return null; } @@ -177,6 +174,7 @@ private C findCounter(T key) { /** * Abstract factory method for new framework counter + * * @param key for the enum value of a counter * @return a new counter for the key */ @@ -248,6 +246,7 @@ private void clear() { public Iterator iterator() { return new AbstractIterator() { int i = 0; + @Override protected C computeNext() { while (i < counters.length) { diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/GenericCounter.java b/tez-api/src/main/java/org/apache/tez/common/counters/GenericCounter.java index 758ab5138e..9752118a29 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/GenericCounter.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/GenericCounter.java @@ -52,7 +52,8 @@ public GenericCounter(String name, String displayName, long value) { this.value.set(value); } - @Override @Deprecated + @Override + @Deprecated public synchronized void setDisplayName(String displayName) { this.displayName = displayName; } @@ -70,7 +71,7 @@ public synchronized void readFields(DataInput in) throws IOException { @Override public synchronized void write(DataOutput out) throws IOException { Text.writeString(out, name); - boolean distinctDisplayName = ! name.equals(displayName); + boolean distinctDisplayName = !name.equals(displayName); out.writeBoolean(distinctDisplayName); if (distinctDisplayName) { Text.writeString(out, displayName); diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/JobCounter.java b/tez-api/src/main/java/org/apache/tez/common/counters/JobCounter.java index 14e76bc081..013b9c9232 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/JobCounter.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/JobCounter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,7 +23,7 @@ // Per-job counters @Private public enum JobCounter { - NUM_FAILED_MAPS, + NUM_FAILED_MAPS, NUM_FAILED_REDUCES, NUM_KILLED_MAPS, NUM_KILLED_REDUCES, diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java b/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java index 3e926c693b..c7347834ce 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java @@ -27,7 +27,7 @@ @InterfaceAudience.Private public class Limits { - + private static final Logger LOG = LoggerFactory.getLogger(Limits.class); private static final Configuration DEFAULT_CONFIGURATION = new TezConfiguration(); @@ -83,8 +83,8 @@ public synchronized void checkCounters(int size) { throw new LimitExceededException(firstViolation); } if (size > COUNTERS_MAX) { - firstViolation = new LimitExceededException("Too many counters: "+ size + - " max="+ COUNTERS_MAX); + firstViolation = new LimitExceededException("Too many counters: " + size + + " max=" + COUNTERS_MAX); throw firstViolation; } } @@ -99,8 +99,8 @@ public synchronized void checkGroups(int size) { throw new LimitExceededException(firstViolation); } if (size > GROUPS_MAX) { - firstViolation = new LimitExceededException("Too many counter groups: "+ - size +" max="+ GROUPS_MAX); + firstViolation = new LimitExceededException("Too many counter groups: " + + size + " max=" + GROUPS_MAX); } } @@ -109,5 +109,4 @@ public synchronized void checkGroups(int size) { public synchronized static void reset() { conf = DEFAULT_CONFIGURATION; } - } diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/TaskCounter.java b/tez-api/src/main/java/org/apache/tez/common/counters/TaskCounter.java index a65388ffd0..cd115b2310 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/TaskCounter.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/TaskCounter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,7 +25,7 @@ @Private public enum TaskCounter { // TODO Eventually, rename counters to be non-MR specific and map them to MR equivalent. - + NUM_SPECULATIONS, /** @@ -39,18 +39,18 @@ public enum TaskCounter { * Alternately number of records seen by a ReduceProcessor */ REDUCE_INPUT_RECORDS, - + REDUCE_OUTPUT_RECORDS, // Not used at the moment. REDUCE_SKIPPED_GROUPS, // Not used at the moment. REDUCE_SKIPPED_RECORDS, // Not used at the moment. SPLIT_RAW_BYTES, - + COMBINE_INPUT_RECORDS, COMBINE_OUTPUT_RECORDS, // Not used at the moment. /** * Number of records written to disk in case of OnFileSortedOutput. - * + * * Number of additional records writtent out to disk in case of * ShuffledMergedInput; this represents the number of unnecessary spills to * disk caused by lac of memory. @@ -61,7 +61,7 @@ public enum TaskCounter { * Number of Inputs from which data is copied. Represents physical Inputs. */ NUM_SHUFFLED_INPUTS, - + /** * Number of Inputs from which data was not copied - typically due to an empty Input */ @@ -71,7 +71,7 @@ public enum TaskCounter { * Number of failed copy attempts (physical inputs) */ NUM_FAILED_SHUFFLE_INPUTS, - + MERGED_MAP_OUTPUTS, GC_TIME_MILLIS, CPU_MILLISECONDS, @@ -84,7 +84,7 @@ public enum TaskCounter { /** * Represents the number of Input Records that were actually processed. * Used by MRInput and ShuffledUnorderedKVInput - * + * */ INPUT_RECORDS_PROCESSED, @@ -112,7 +112,7 @@ public enum TaskCounter { * spilled directly */ OUTPUT_LARGE_RECORDS, - + SKIPPED_RECORDS, // Not used at the moment. /** @@ -132,19 +132,19 @@ public enum TaskCounter { * size + overhead) */ OUTPUT_BYTES_PHYSICAL, - + /** * Bytes written to disk due to unnecessary spills (lac of adequate memory). * Used by OnFileSortedOutput and ShuffledMergedInput */ ADDITIONAL_SPILLS_BYTES_WRITTEN, - + /** * Bytes read from disk due to previous spills (lac of adequate memory). * Used by OnFileSortedOutput and ShuffledMergedInput */ ADDITIONAL_SPILLS_BYTES_READ, - + /** * Spills that were generated & read by the same task (unnecessary spills due to lac of * adequate memory). @@ -159,7 +159,7 @@ public enum TaskCounter { * as final merge is avoided. */ SHUFFLE_CHUNK_COUNT, - + INPUT_GROUPS, // Not used at the moment. Will eventually replace REDUCE_INPUT_GROUPS /** @@ -172,7 +172,7 @@ public enum TaskCounter { * Uncompressed size of the data being processed by the relevant Shuffle. * Includes serialization, file format etc overheads. */ - SHUFFLE_BYTES_DECOMPRESSED, + SHUFFLE_BYTES_DECOMPRESSED, /** * Number of bytes which were shuffled directly to memory. @@ -230,11 +230,10 @@ public enum TaskCounter { */ LAST_EVENT_RECEIVED, - /** * The size of the data that is transmitted via event. * * Represented in number of bytes */ DATA_BYTES_VIA_EVENT -} \ No newline at end of file +} diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/TezCounter.java b/tez-api/src/main/java/org/apache/tez/common/counters/TezCounter.java index 4cb1ae94e0..1f5ca2ecd6 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/TezCounter.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/TezCounter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -40,7 +40,7 @@ public interface TezCounter extends Writable { /** * Set the display name of the counter * @param displayName of the counter - * @deprecated (and no-op by default) + * @deprecated (and no - op by default) */ @Deprecated void setDisplayName(String displayName); @@ -80,8 +80,10 @@ public interface TezCounter extends Writable { */ public default void aggregate(TezCounter other) { increment(other.getValue()); - }; - + } + + ; + /** * Return the underlying object if this is a facade. * @return the undelying object. diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/TezCounters.java b/tez-api/src/main/java/org/apache/tez/common/counters/TezCounters.java index a1205b9fdd..d0c012f8f7 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/TezCounters.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/TezCounters.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -102,9 +102,10 @@ private static class GroupFactory @Override protected > FrameworkGroupFactory - newFrameworkGroupFactory(final Class cls) { + newFrameworkGroupFactory(final Class cls) { return new FrameworkGroupFactory() { - @Override public CounterGroup newGroup(String name) { + @Override + public CounterGroup newGroup(String name) { return new FrameworkGroupImpl(cls); // impl in this package } }; diff --git a/tez-api/src/main/java/org/apache/tez/common/security/ACLConfigurationParser.java b/tez-api/src/main/java/org/apache/tez/common/security/ACLConfigurationParser.java index d788a46c16..34d6ca83bb 100644 --- a/tez-api/src/main/java/org/apache/tez/common/security/ACLConfigurationParser.java +++ b/tez-api/src/main/java/org/apache/tez/common/security/ACLConfigurationParser.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -61,7 +61,6 @@ public ACLConfigurationParser(Configuration conf, boolean dagACLs) { parse(dagACLs); } - private void parse(boolean dagACLs) { if (!dagACLs) { parseACLType(YarnConfiguration.YARN_ADMIN_ACL, ACLType.YARN_ADMIN_ACL); @@ -148,5 +147,4 @@ public void addAllowedGroups(Map> additionalAllowedGroups) } } } - } diff --git a/tez-api/src/main/java/org/apache/tez/common/security/ACLManager.java b/tez-api/src/main/java/org/apache/tez/common/security/ACLManager.java index 08680a5b50..1f0a99a238 100644 --- a/tez-api/src/main/java/org/apache/tez/common/security/ACLManager.java +++ b/tez-api/src/main/java/org/apache/tez/common/security/ACLManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -214,7 +214,7 @@ public Map toYARNACls() { if (groups != null && !groups.isEmpty()) { for (Entry> entry : groups.entrySet()) { if (entry.getKey().equals(ACLType.AM_VIEW_ACL) - && !viewAclsWildCard && !entry.getValue().isEmpty()) { + && !viewAclsWildCard && !entry.getValue().isEmpty()) { // Append groups only if wild card not set String aclsStr = acls.containsKey(ApplicationAccessType.VIEW_APP) ? acls.get(ApplicationAccessType.VIEW_APP) : ""; @@ -247,5 +247,3 @@ public static String toCommaSeparatedString(Collection collection) { return sb.toString(); } } - - diff --git a/tez-api/src/main/java/org/apache/tez/common/security/ACLType.java b/tez-api/src/main/java/org/apache/tez/common/security/ACLType.java index 0202e1bc04..12d55101c6 100644 --- a/tez-api/src/main/java/org/apache/tez/common/security/ACLType.java +++ b/tez-api/src/main/java/org/apache/tez/common/security/ACLType.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-api/src/main/java/org/apache/tez/common/security/DAGAccessControls.java b/tez-api/src/main/java/org/apache/tez/common/security/DAGAccessControls.java index 520416d576..ed44aacd93 100644 --- a/tez-api/src/main/java/org/apache/tez/common/security/DAGAccessControls.java +++ b/tez-api/src/main/java/org/apache/tez/common/security/DAGAccessControls.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -84,7 +84,6 @@ public DAGAccessControls(String viewACLsStr, String modifyACLsStr) { if (allowedGroups.containsKey(ACLType.DAG_MODIFY_ACL)) { this.groupsWithModifyACLs.addAll(allowedGroups.get(ACLType.DAG_MODIFY_ACL)); } - } /** diff --git a/tez-api/src/main/java/org/apache/tez/common/security/DAGClientSecurityInfo.java b/tez-api/src/main/java/org/apache/tez/common/security/DAGClientSecurityInfo.java index a522c6a8e7..c280274b37 100644 --- a/tez-api/src/main/java/org/apache/tez/common/security/DAGClientSecurityInfo.java +++ b/tez-api/src/main/java/org/apache/tez/common/security/DAGClientSecurityInfo.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.common.security; @@ -52,7 +52,7 @@ public Class annotationType() { @Override public Class> - value() { + value() { return ClientToAMTokenSelector.class; } }; diff --git a/tez-api/src/main/java/org/apache/tez/common/security/HistoryACLPolicyException.java b/tez-api/src/main/java/org/apache/tez/common/security/HistoryACLPolicyException.java index 80229bc34e..d535331017 100644 --- a/tez-api/src/main/java/org/apache/tez/common/security/HistoryACLPolicyException.java +++ b/tez-api/src/main/java/org/apache/tez/common/security/HistoryACLPolicyException.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,9 +26,12 @@ @Public public class HistoryACLPolicyException extends Exception { private static final long serialVersionUID = 5337242734701961239L; - public HistoryACLPolicyException(Throwable cause) { super(cause); } - public HistoryACLPolicyException(String message) { super(message); } + + public HistoryACLPolicyException(Throwable cause) {super(cause);} + + public HistoryACLPolicyException(String message) {super(message);} + public HistoryACLPolicyException(String message, Throwable cause) { super(message, cause); } -} \ No newline at end of file +} diff --git a/tez-api/src/main/java/org/apache/tez/common/security/HistoryACLPolicyManager.java b/tez-api/src/main/java/org/apache/tez/common/security/HistoryACLPolicyManager.java index 92eea67440..becb2ff44f 100644 --- a/tez-api/src/main/java/org/apache/tez/common/security/HistoryACLPolicyManager.java +++ b/tez-api/src/main/java/org/apache/tez/common/security/HistoryACLPolicyManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -60,7 +60,7 @@ public Map setupSessionACLs(Configuration conf, ApplicationId ap * @throws Exception */ public Map setupNonSessionACLs(Configuration conf, ApplicationId applicationId, - DAGAccessControls dagAccessControls) throws IOException, HistoryACLPolicyException; + DAGAccessControls dagAccessControls) throws IOException, HistoryACLPolicyException; /** * Take any necessary steps for setting up ACLs for a DAG that is submitted to a Session. This is @@ -75,8 +75,8 @@ public Map setupNonSessionACLs(Configuration conf, ApplicationId * @throws Exception */ public Map setupSessionDAGACLs(Configuration conf, ApplicationId applicationId, - String dagName, DAGAccessControls dagAccessControls) - throws IOException, HistoryACLPolicyException; + String dagName, DAGAccessControls dagAccessControls) + throws IOException, HistoryACLPolicyException; /** * Called with a timeline entity which has to be updated with a domain id. diff --git a/tez-api/src/main/java/org/apache/tez/common/security/JobTokenIdentifier.java b/tez-api/src/main/java/org/apache/tez/common/security/JobTokenIdentifier.java index 47bf74d2ee..dca7fe1911 100644 --- a/tez-api/src/main/java/org/apache/tez/common/security/JobTokenIdentifier.java +++ b/tez-api/src/main/java/org/apache/tez/common/security/JobTokenIdentifier.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -37,7 +37,7 @@ public class JobTokenIdentifier extends TokenIdentifier { private Text jobid; public final static Text KIND_NAME = new Text("tez.job"); - + /** * Default constructor */ @@ -58,7 +58,7 @@ public JobTokenIdentifier(Text jobid) { public Text getKind() { return KIND_NAME; } - + /** {@inheritDoc} */ @Override public UserGroupInformation getUser() { @@ -67,7 +67,7 @@ public UserGroupInformation getUser() { } return UserGroupInformation.createRemoteUser(jobid.toString()); } - + /** * Get the jobid * @return the jobid diff --git a/tez-api/src/main/java/org/apache/tez/common/security/JobTokenSecretManager.java b/tez-api/src/main/java/org/apache/tez/common/security/JobTokenSecretManager.java index 785613e928..80f5f7e56c 100644 --- a/tez-api/src/main/java/org/apache/tez/common/security/JobTokenSecretManager.java +++ b/tez-api/src/main/java/org/apache/tez/common/security/JobTokenSecretManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -67,7 +67,7 @@ public static byte[] computeHash(byte[] msg, SecretKey key) { * @return the computed hash */ public byte[] computeHash(byte[] msg) { - synchronized(mac) { + synchronized (mac) { return mac.doFinal(msg); } } @@ -92,7 +92,6 @@ public JobTokenSecretManager(SecretKey key) { } } - /** * Create a new password/secret for the given job token identifier. * @param identifier the job token identifier @@ -124,7 +123,7 @@ public void removeTokenForJob(String jobId) { currentJobTokens.remove(jobId); } } - + /** * Look up the token password/secret for the given jobId. * @param jobId the jobId to look up @@ -141,7 +140,7 @@ public SecretKey retrieveTokenSecret(String jobId) throws InvalidToken { } return tokenSecret; } - + /** * Look up the token password/secret for the given job token identifier. * @param identifier the job token identifier to look up diff --git a/tez-api/src/main/java/org/apache/tez/common/security/Master.java b/tez-api/src/main/java/org/apache/tez/common/security/Master.java index de73d10092..d968d02476 100644 --- a/tez-api/src/main/java/org/apache/tez/common/security/Master.java +++ b/tez-api/src/main/java/org/apache/tez/common/security/Master.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -39,7 +39,6 @@ public static String getMasterUserName(Configuration conf) { return conf.get(YarnConfiguration.RM_PRINCIPAL); } - // This needs to go into YARN public static InetSocketAddress getMasterAddress(Configuration conf) { return conf @@ -55,5 +54,4 @@ public static String getMasterPrincipal(Configuration conf) return SecurityUtil.getServerPrincipal(getMasterUserName(conf), masterHostname); } - } diff --git a/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java b/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java index e56ef61270..775979bf40 100644 --- a/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java +++ b/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,7 +36,6 @@ import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.tez.dag.api.TezConfiguration; - /** * This class provides user facing APIs for transferring secrets from * the job client to the tasks. @@ -46,21 +45,20 @@ @InterfaceAudience.Private @InterfaceStability.Unstable public class TokenCache { - + private static final Logger LOG = LoggerFactory.getLogger(TokenCache.class); - /** * auxiliary method to get user's secret keys.. * @param alias * @return secret key from the storage */ public static byte[] getSecretKey(Credentials credentials, Text alias) { - if(credentials == null) + if (credentials == null) return null; return credentials.getSecretKey(alias); } - + /** * Convenience method to obtain delegation tokens from namenodes * corresponding to the paths passed. @@ -70,7 +68,7 @@ public static byte[] getSecretKey(Credentials credentials, Text alias) { * @throws IOException */ public static void obtainTokensForFileSystems(Credentials credentials, - Path[] ps, Configuration conf) throws IOException { + Path[] ps, Configuration conf) throws IOException { if (!UserGroupInformation.isSecurityEnabled()) { return; } @@ -78,11 +76,12 @@ public static void obtainTokensForFileSystems(Credentials credentials, } private static final int MAX_FS_OBJECTS = 10; + static void obtainTokensForFileSystemsInternal(Credentials credentials, - Path[] ps, Configuration conf) throws IOException { + Path[] ps, Configuration conf) throws IOException { Set fsSet = new HashSet(); boolean limitExceeded = false; - for(Path p: ps) { + for (Path p : ps) { FileSystem fs = p.getFileSystem(conf); if (!limitExceeded && fsSet.size() == MAX_FS_OBJECTS) { LOG.warn("No of FileSystem objects exceeds {}, updating tokens for all paths. This can" + @@ -104,10 +103,10 @@ static void obtainTokensForFileSystemsInternal(Credentials credentials, static boolean isTokenRenewalExcluded(FileSystem fs, Configuration conf) { String[] nns = - conf.getStrings(TezConfiguration.TEZ_JOB_FS_SERVERS_TOKEN_RENEWAL_EXCLUDE); + conf.getStrings(TezConfiguration.TEZ_JOB_FS_SERVERS_TOKEN_RENEWAL_EXCLUDE); if (nns != null) { String host = fs.getUri().getHost(); - for(int i = 0; i < nns.length; i++) { + for (int i = 0; i < nns.length; i++) { if (nns[i].equals(host)) { return true; } @@ -124,8 +123,8 @@ static boolean isTokenRenewalExcluded(FileSystem fs, Configuration conf) { * @param conf * @throws IOException */ - static void obtainTokensForFileSystemsInternal(FileSystem fs, - Credentials credentials, Configuration conf) throws IOException { + static void obtainTokensForFileSystemsInternal(FileSystem fs, + Credentials credentials, Configuration conf) throws IOException { // TODO Change this to use YARN utilities once YARN-1664 is fixed. // RM skips renewing token with empty renewer String delegTokenRenewer = ""; @@ -133,15 +132,15 @@ static void obtainTokensForFileSystemsInternal(FileSystem fs, delegTokenRenewer = Master.getMasterPrincipal(conf); if (delegTokenRenewer == null || delegTokenRenewer.length() == 0) { throw new IOException( - "Can't get Master Kerberos principal for use as renewer"); + "Can't get Master Kerberos principal for use as renewer"); } } final Token tokens[] = fs.addDelegationTokens(delegTokenRenewer, - credentials); + credentials); if (tokens != null) { for (Token token : tokens) { - LOG.info("Got dt for " + fs.getUri() + "; "+token); + LOG.info("Got dt for " + fs.getUri() + "; " + token); } } } @@ -153,12 +152,13 @@ static void obtainTokensForFileSystemsInternal(FileSystem fs, * @param t */ @InterfaceAudience.Private - public static void setSessionToken(Token t, - Credentials credentials) { + public static void setSessionToken(Token t, + Credentials credentials) { credentials.addToken(SESSION_TOKEN, t); } + /** - * + * * @return session token */ @SuppressWarnings("unchecked") @@ -178,7 +178,7 @@ public static Token getSessionToken(Credentials credentials) */ @InterfaceAudience.Private public static void mergeBinaryTokens(Credentials creds, - Configuration conf, String tokenFilePath) + Configuration conf, String tokenFilePath) throws IOException { if (tokenFilePath == null || tokenFilePath.isEmpty()) { throw new RuntimeException("Invalid file path provided" @@ -187,10 +187,9 @@ public static void mergeBinaryTokens(Credentials creds, LOG.info("Merging additional tokens from binary file" + ", binaryFileName=" + tokenFilePath); Credentials binary = Credentials.readTokenStorageFile( - new Path("file:///" + tokenFilePath), conf); + new Path("file:///" + tokenFilePath), conf); // supplement existing tokens with the tokens in the binary file creds.mergeAll(binary); } - } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/ConfigurationScope.java b/tez-api/src/main/java/org/apache/tez/dag/api/ConfigurationScope.java index bf2a7d9a48..8387168eb5 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/ConfigurationScope.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/ConfigurationScope.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java index 0864b82e80..9b70152d01 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -104,7 +104,7 @@ public class DAG { Map commonTaskLocalFiles = Maps.newHashMap(); String dagInfo; CallerContext callerContext; - private Map dagConf = new HashMap(); + private Map dagConf = new HashMap(); private VertexExecutionContext defaultExecutionContext; private DAG(String name) { @@ -137,7 +137,7 @@ public synchronized DAG addTaskLocalFiles(Map localFiles) public synchronized DAG addVertex(Vertex vertex) { if (vertices.containsKey(vertex.getName())) { throw new IllegalStateException( - "Vertex " + vertex.getName() + " already defined!"); + "Vertex " + vertex.getName() + " already defined!"); } vertices.put(vertex.getName(), vertex); return this; @@ -184,7 +184,6 @@ public synchronized DAG setDAGInfo(String dagInfo) { return this; } - /** * Set the Context in which Tez is being called. * @param callerContext Caller Context @@ -219,7 +218,6 @@ public synchronized Credentials getCredentials() { return this.credentials; } - /** * Set Access controls for the DAG. Which user/groups can view the DAG progess/history and * who can modify the DAG i.e. kill the DAG. @@ -287,15 +285,15 @@ public synchronized DAG addEdge(Edge edge) { // Sanity checks if (!vertices.containsValue(edge.getInputVertex())) { throw new IllegalArgumentException( - "Input vertex " + edge.getInputVertex() + " doesn't exist!"); + "Input vertex " + edge.getInputVertex() + " doesn't exist!"); } if (!vertices.containsValue(edge.getOutputVertex())) { throw new IllegalArgumentException( - "Output vertex " + edge.getOutputVertex() + " doesn't exist!"); + "Output vertex " + edge.getOutputVertex() + " doesn't exist!"); } if (edges.contains(edge)) { throw new IllegalArgumentException( - "Edge " + edge + " already defined!"); + "Edge " + edge + " already defined!"); } // inform the vertices @@ -315,15 +313,15 @@ public synchronized DAG addEdge(GroupInputEdge edge) { // Sanity checks if (!vertexGroups.contains(edge.getInputVertexGroup())) { throw new IllegalArgumentException( - "Input vertex " + edge.getInputVertexGroup() + " doesn't exist!"); + "Input vertex " + edge.getInputVertexGroup() + " doesn't exist!"); } if (!vertices.containsValue(edge.getOutputVertex())) { throw new IllegalArgumentException( - "Output vertex " + edge.getOutputVertex() + " doesn't exist!"); + "Output vertex " + edge.getOutputVertex() + " doesn't exist!"); } if (groupInputEdges.contains(edge)) { throw new IllegalArgumentException( - "GroupInputEdge " + edge + " already defined!"); + "GroupInputEdge " + edge + " already defined!"); } VertexGroup av = edge.getInputVertexGroup(); @@ -414,7 +412,7 @@ VertexExecutionContext getDefaultExecutionContext() { @Private @VisibleForTesting - public Map getDagConf() { + public Map getDagConf() { return dagConf; } @@ -439,7 +437,7 @@ void checkAndInferOneToOneParallelism() { // add newly inferred vertices for consideration as known sources // the outer loop will run for every new level of inferring the parallelism // however, the entire logic will process each vertex only once - while(!newKnownTasksVertices.isEmpty()) { + while (!newKnownTasksVertices.isEmpty()) { Set knownTasksVertices = Sets.newHashSet(newKnownTasksVertices); newKnownTasksVertices.clear(); for (Vertex v : knownTasksVertices) { @@ -469,8 +467,8 @@ void checkAndInferOneToOneParallelism() { if (outputVertex.getParallelism() != -1) { throw new TezUncheckedException( "1-1 Edge. Destination vertex parallelism must match source vertex. " - + "Vertex: " + inputVertex.getName() + " does not match vertex: " - + outputVertex.getName()); + + "Vertex: " + inputVertex.getName() + " does not match vertex: " + + outputVertex.getName()); } } } @@ -501,13 +499,13 @@ void checkAndInferOneToOneParallelism() { // and as a result, an initializer is not needed. if (vertex.getDataSources() != null && vertex.getDataSources().size() == 1 - && vertex.getDataSources().get(0).getNumberOfShards() > -1) { + && vertex.getDataSources().get(0).getNumberOfShards() > -1) { continue; } } boolean has1to1UninitedSources = false; - if (vertex.getInputVertices()!= null && !vertex.getInputVertices().isEmpty()) { + if (vertex.getInputVertices() != null && !vertex.getInputVertices().isEmpty()) { for (Vertex srcVertex : vertex.getInputVertices()) { if (srcVertex.getParallelism() == -1) { has1to1UninitedSources = true; @@ -537,7 +535,6 @@ private static class AnnotatedVertex { int lowlink; //for Tarjan's algorithm boolean onstack; //for Tarjan's algorithm - private AnnotatedVertex(Vertex v) { this.v = v; index = -1; @@ -583,7 +580,7 @@ Deque verify(boolean restricted) throws IllegalStateException { for (Vertex v : vertices.values()) { if (vertexMap.containsKey(v.getName())) { throw new IllegalStateException("DAG contains multiple vertices" - + " with name: " + v.getName()); + + " with name: " + v.getName()); } vertexMap.put(v.getName(), new AnnotatedVertex(v)); } @@ -620,7 +617,7 @@ Deque verify(boolean restricted) throws IllegalStateException { // check input and output names don't collide with vertex names for (Vertex vertex : vertices.values()) { for (RootInputLeafOutput - input : vertex.getInputs()) { + input : vertex.getInputs()) { if (vertexMap.containsKey(input.getName())) { throw new IllegalStateException("Vertex: " + vertex.getName() @@ -629,7 +626,7 @@ Deque verify(boolean restricted) throws IllegalStateException { } } for (RootInputLeafOutput - output : vertex.getOutputs()) { + output : vertex.getOutputs()) { if (vertexMap.containsKey(output.getName())) { throw new IllegalStateException("Vertex: " + vertex.getName() @@ -643,7 +640,7 @@ Deque verify(boolean restricted) throws IllegalStateException { for (Entry> entry : inboundVertexMap.entrySet()) { Vertex vertex = entry.getKey(); for (RootInputLeafOutput - input : vertex.getInputs()) { + input : vertex.getInputs()) { if (entry.getValue().contains(input.getName())) { throw new IllegalStateException("Vertex: " + vertex.getName() @@ -657,7 +654,7 @@ Deque verify(boolean restricted) throws IllegalStateException { for (Entry> entry : outboundVertexMap.entrySet()) { Vertex vertex = entry.getKey(); for (RootInputLeafOutput - output : vertex.getOutputs()) { + output : vertex.getOutputs()) { if (entry.getValue().contains(output.getName())) { throw new IllegalStateException("Vertex: " + vertex.getName() @@ -667,7 +664,6 @@ Deque verify(boolean restricted) throws IllegalStateException { } } - // Not checking for repeated input names / output names vertex names on the same vertex, // since we only allow 1 at the moment. // When additional inputs are supported, this can be chceked easily (and early) @@ -683,14 +679,13 @@ Deque verify(boolean restricted) throws IllegalStateException { if (dataSourceType != DataSourceType.PERSISTED && dataSourceType != DataSourceType.EPHEMERAL) { throw new IllegalStateException( - "Unsupported source type on edge. " + e); + "Unsupported source type on edge. " + e); } } } // check for conflicts between dag level local resource and vertex level local resource - return topologicalVertexStack; } @@ -738,7 +733,6 @@ void verifyLocalResources(Configuration tezConf) { commonTaskLocalFiles.get(resourceName) + "\nResource of vertex: " + resource); } - } catch (URISyntaxException | IOException e) { throw new RuntimeException( "Failed while attempting to validate sha for conflicting resources (" + @@ -757,8 +751,8 @@ void verifyLocalResources(Configuration tezConf) { // Adaptation of Tarjan's algorithm for connected components. // http://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm private Deque detectCycles(Map> edgeMap, - Map vertexMap) - throws IllegalStateException { + Map vertexMap) + throws IllegalStateException { Deque topologicalVertexStack = new LinkedList(); Integer nextIndex = 0; // boxed integer so it is passed by reference. Stack stack = new Stack(); @@ -841,9 +835,9 @@ public DAGPlan createDag(Configuration tezConf, Credentials extraCredentials, // create protobuf message describing DAG @Private public synchronized DAGPlan createDag(Configuration tezConf, Credentials extraCredentials, - Map tezJarResources, LocalResource binaryConfig, - boolean tezLrsAsArchive, ServicePluginsDescriptor servicePluginsDescriptor, - JavaOptsChecker javaOptsChecker) { + Map tezJarResources, LocalResource binaryConfig, + boolean tezLrsAsArchive, ServicePluginsDescriptor servicePluginsDescriptor, + JavaOptsChecker javaOptsChecker) { Deque topologicalVertexStack = verify(true); verifyLocalResources(tezConf); @@ -876,10 +870,10 @@ public synchronized DAGPlan createDag(Configuration tezConf, Credentials extraCr } groupBuilder.addAllOutputs(groupInfo.outputs); for (Map.Entry entry : - groupInfo.edgeMergedInputs.entrySet()) { + groupInfo.edgeMergedInputs.entrySet()) { groupBuilder.addEdgeMergedInputs( PlanGroupInputEdgeInfo.newBuilder().setDestVertexName(entry.getKey()). - setMergedInput(DagTypeConverters.convertToDAGPlan(entry.getValue()))); + setMergedInput(DagTypeConverters.convertToDAGPlan(entry.getValue()))); } dagBuilder.addVertexGroups(groupBuilder); } @@ -896,9 +890,9 @@ public synchronized DAGPlan createDag(Configuration tezConf, Credentials extraCr Preconditions.checkArgument(topologicalVertexStack.size() == vertices.size(), "size of topologicalVertexStack is:" + topologicalVertexStack.size() + - " while size of vertices is:" + vertices.size() + - ", make sure they are the same in order to sort the vertices"); - while(!topologicalVertexStack.isEmpty()) { + " while size of vertices is:" + vertices.size() + + ", make sure they are the same in order to sort the vertices"); + while (!topologicalVertexStack.isEmpty()) { Vertex vertex = vertices.get(topologicalVertexStack.pop()); // infer credentials, resources and parallelism from data source Resource vertexTaskResource = vertex.getTaskResource(); @@ -981,7 +975,7 @@ public synchronized DAGPlan createDag(Configuration tezConf, Credentials extraCr } } - if (vertex.getConf()!= null && vertex.getConf().size() > 0) { + if (vertex.getConf() != null && vertex.getConf().size() > 0) { ConfigurationProto.Builder confBuilder = ConfigurationProto.newBuilder(); TezUtils.populateConfProtoFromEntries(vertex.getConf().entrySet(), confBuilder); vertexBuilder.setVertexConf(confBuilder); @@ -1094,7 +1088,7 @@ public synchronized DAGPlan createDag(Configuration tezConf, Credentials extraCr if (!HistoryLogLevel.validateLogLevel(logLevel)) { throw new IllegalArgumentException( "Config: " + TezConfiguration.TEZ_HISTORY_LOGGING_LOGLEVEL + - " is set to invalid value: " + logLevel); + " is set to invalid value: " + logLevel); } } else { // Validate and set value from tezConf. @@ -1103,7 +1097,7 @@ public synchronized DAGPlan createDag(Configuration tezConf, Credentials extraCr if (!HistoryLogLevel.validateLogLevel(logLevel)) { throw new IllegalArgumentException( "Config: " + TezConfiguration.TEZ_HISTORY_LOGGING_LOGLEVEL + - " is set to invalid value: " + logLevel); + " is set to invalid value: " + logLevel); } PlanKeyValuePair.Builder kvp = PlanKeyValuePair.newBuilder(); kvp.setKey(TezConfiguration.TEZ_HISTORY_LOGGING_LOGLEVEL); @@ -1191,5 +1185,4 @@ private boolean checkNamedEntityExists(String expected, NamedEntityDescriptor[] public synchronized CallerContext getCallerContext() { return this.callerContext; } - } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAGNotRunningException.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAGNotRunningException.java index cbc93a9647..30c7c202ee 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/DAGNotRunningException.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAGNotRunningException.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.api; @@ -26,8 +26,11 @@ @Private public class DAGNotRunningException extends TezException { private static final long serialVersionUID = 6337442733802964448L; - public DAGNotRunningException(Throwable cause) { super(cause); } - public DAGNotRunningException(String message) { super(message); } + + public DAGNotRunningException(Throwable cause) {super(cause);} + + public DAGNotRunningException(String message) {super(message);} + public DAGNotRunningException(String message, Throwable cause) { super(message, cause); } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAGSubmissionTimedOut.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAGSubmissionTimedOut.java index 347ed3d4a0..341c209f74 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/DAGSubmissionTimedOut.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAGSubmissionTimedOut.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,5 +31,4 @@ public class DAGSubmissionTimedOut extends TezException { public DAGSubmissionTimedOut(String message) { super(message); } - } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java b/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java index c563f1fb47..3e16d6276d 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -92,15 +92,19 @@ @Private public class DagTypeConverters { - public static PlanLocalResourceVisibility convertToDAGPlan(LocalResourceVisibility visibility){ - switch(visibility){ - case PUBLIC : return PlanLocalResourceVisibility.PUBLIC; - case PRIVATE : return PlanLocalResourceVisibility.PRIVATE; - case APPLICATION : return PlanLocalResourceVisibility.APPLICATION; - default : throw new RuntimeException("unknown 'visibility': " + visibility); + public static PlanLocalResourceVisibility convertToDAGPlan(LocalResourceVisibility visibility) { + switch (visibility) { + case PUBLIC: + return PlanLocalResourceVisibility.PUBLIC; + case PRIVATE: + return PlanLocalResourceVisibility.PRIVATE; + case APPLICATION: + return PlanLocalResourceVisibility.APPLICATION; + default: + throw new RuntimeException("unknown 'visibility': " + visibility); } } - + public static List convertToDAGPlan(Map lrs) { List planLrs = Lists.newArrayListWithCapacity(lrs.size()); for (Entry entry : lrs.entrySet()) { @@ -109,17 +113,17 @@ public static List convertToDAGPlan(Map convertToDAGPlan(Map outputList = new ArrayList(); - for(PlanTaskLocationHint inputHint : locationHints){ + for (PlanTaskLocationHint inputHint : locationHints) { TaskLocationHint outputHint = TaskLocationHint.createTaskLocationHint( new HashSet(inputHint.getHostList()), new HashSet(inputHint.getRackList())); @@ -222,7 +262,7 @@ public static VertexLocationHint convertFromDAGPlan( } return VertexLocationHint.create(outputList); } - + public static String convertToDAGPlan(URL resource) { Path p; try { @@ -242,12 +282,12 @@ public static URL convertToYarnURL(String pathString) { public static Map createLocalResourceMapFromDAGPlan( List localResourcesList) { Map map = new HashMap(); - for(PlanLocalResource res : localResourcesList){ + for (PlanLocalResource res : localResourcesList) { LocalResource r = new LocalResourcePBImpl(); //NOTE: have to check every optional field in protobuf generated classes for existence before accessing //else we will receive a default value back, eg "" - if(res.hasPattern()){ + if (res.hasPattern()) { r.setPattern(res.getPattern()); } r.setResource(convertToYarnURL(res.getUri())); @@ -264,22 +304,22 @@ public static Map createEnvironmentMapFromDAGPlan( List environmentSettingList) { Map map = new HashMap(); - for(PlanKeyValuePair setting : environmentSettingList){ + for (PlanKeyValuePair setting : environmentSettingList) { map.put(setting.getKey(), setting.getValue()); } return map; } - public static Map createEdgePlanMapFromDAGPlan(List edgeList){ + public static Map createEdgePlanMapFromDAGPlan(List edgeList) { Map edgePlanMap = new HashMap(); - for(EdgePlan edgePlanItem : edgeList){ + for (EdgePlan edgePlanItem : edgeList) { edgePlanMap.put(edgePlanItem.getId(), edgePlanItem); } return edgePlanMap; } - + public static PlanEdgeProperty convertToProto(EdgeProperty prop) { PlanEdgeProperty.Builder edgePropBuilder = PlanEdgeProperty.newBuilder(); edgePropBuilder.setDataMovementType(convertToDAGPlan(prop.getDataMovementType())); @@ -292,20 +332,20 @@ public static PlanEdgeProperty convertToProto(EdgeProperty prop) { edgePropBuilder.setEdgeManager(DagTypeConverters.convertToDAGPlan(prop .getEdgeManagerDescriptor())); } - + return edgePropBuilder.build(); } - + public static EdgeProperty convertFromProto(PlanEdgeProperty edge) { - return EdgeProperty.create( - (edge.hasEdgeManager() ? - convertEdgeManagerPluginDescriptorFromDAGPlan(edge.getEdgeManager()) : null), - convertFromDAGPlan(edge.getDataMovementType()), - convertFromDAGPlan(edge.getDataSourceType()), - convertFromDAGPlan(edge.getSchedulingType()), - convertOutputDescriptorFromDAGPlan(edge.getEdgeSource()), - convertInputDescriptorFromDAGPlan(edge.getEdgeDestination()) - ); + return EdgeProperty.create( + (edge.hasEdgeManager() ? + convertEdgeManagerPluginDescriptorFromDAGPlan(edge.getEdgeManager()) : null), + convertFromDAGPlan(edge.getDataMovementType()), + convertFromDAGPlan(edge.getDataSourceType()), + convertFromDAGPlan(edge.getSchedulingType()), + convertOutputDescriptorFromDAGPlan(edge.getEdgeSource()), + convertInputDescriptorFromDAGPlan(edge.getEdgeDestination()) + ); } public static EdgeProperty createEdgePropertyMapFromDAGPlan(EdgePlan edge) { @@ -338,7 +378,7 @@ public static Map convertConfFromProto( ConfigurationProto confProto) { List settingList = confProto.getConfKeyValuesList(); Map map = new HashMap(); - for(PlanKeyValuePair setting: settingList){ + for (PlanKeyValuePair setting : settingList) { map.put(setting.getKey(), setting.getValue()); } return map; @@ -382,8 +422,6 @@ public static String getHistoryTextFromProto(TezEntityDescriptorProto proto, Inf } } - - public static RootInputLeafOutputProto convertToDAGPlan( RootInputLeafOutput, ? extends EntityDescriptor> rootIO) { RootInputLeafOutputProto.Builder builder = RootInputLeafOutputProto.newBuilder(); @@ -409,8 +447,6 @@ private static UserPayload convertTezUserPayloadFromDAGPlan( return userPayload; } - - private static void setUserPayload(EntityDescriptor entity, UserPayload payload) { if (payload != null) { entity.setUserPayload(payload); @@ -490,42 +526,41 @@ public static ProcessorDescriptor convertProcessorDescriptorFromDAGPlan( } public static TezAppMasterStatus convertTezAppMasterStatusFromProto( - TezAppMasterStatusProto proto) { + TezAppMasterStatusProto proto) { switch (proto) { - case INITIALIZING: - return TezAppMasterStatus.INITIALIZING; - case READY: - return TezAppMasterStatus.READY; - case RUNNING: - return TezAppMasterStatus.RUNNING; - case SHUTDOWN: - return TezAppMasterStatus.SHUTDOWN; + case INITIALIZING: + return TezAppMasterStatus.INITIALIZING; + case READY: + return TezAppMasterStatus.READY; + case RUNNING: + return TezAppMasterStatus.RUNNING; + case SHUTDOWN: + return TezAppMasterStatus.SHUTDOWN; } throw new TezUncheckedException("Could not convert to TezSessionStatus from" + " proto"); } public static TezAppMasterStatusProto convertTezAppMasterStatusToProto( - TezAppMasterStatus status) { + TezAppMasterStatus status) { switch (status) { - case INITIALIZING: - return TezAppMasterStatusProto.INITIALIZING; - case READY: - return TezAppMasterStatusProto.READY; - case RUNNING: - return TezAppMasterStatusProto.RUNNING; - case SHUTDOWN: - return TezAppMasterStatusProto.SHUTDOWN; + case INITIALIZING: + return TezAppMasterStatusProto.INITIALIZING; + case READY: + return TezAppMasterStatusProto.READY; + case RUNNING: + return TezAppMasterStatusProto.RUNNING; + case SHUTDOWN: + return TezAppMasterStatusProto.SHUTDOWN; } throw new TezUncheckedException("Could not convert TezSessionStatus to" + " proto"); } - public static PlanLocalResourcesProto convertFromLocalResources( - Map localResources) { + Map localResources) { PlanLocalResourcesProto.Builder builder = - PlanLocalResourcesProto.newBuilder(); + PlanLocalResourcesProto.newBuilder(); for (Map.Entry entry : localResources.entrySet()) { PlanLocalResource plr = convertLocalResourceToPlanLocalResource( entry.getKey(), entry.getValue()); @@ -535,9 +570,9 @@ public static PlanLocalResourcesProto convertFromLocalResources( } public static Map convertFromPlanLocalResources( - PlanLocalResourcesProto proto) { + PlanLocalResourcesProto proto) { Map localResources = - new HashMap(proto.getLocalResourcesCount()); + new HashMap(proto.getLocalResourcesCount()); for (PlanLocalResource plr : proto.getLocalResourcesList()) { String name = plr.getName(); LocalResource lr = convertPlanLocalResourceToLocalResource(plr); @@ -547,21 +582,21 @@ public static Map convertFromPlanLocalResources( } public static PlanLocalResource convertLocalResourceToPlanLocalResource( - String name, LocalResource lr) { + String name, LocalResource lr) { PlanLocalResource.Builder localResourcesBuilder = PlanLocalResource.newBuilder(); localResourcesBuilder.setName(name); localResourcesBuilder.setUri( - DagTypeConverters.convertToDAGPlan(lr.getResource())); + DagTypeConverters.convertToDAGPlan(lr.getResource())); localResourcesBuilder.setSize(lr.getSize()); localResourcesBuilder.setTimeStamp(lr.getTimestamp()); localResourcesBuilder.setType( - DagTypeConverters.convertToDAGPlan(lr.getType())); + DagTypeConverters.convertToDAGPlan(lr.getType())); localResourcesBuilder.setVisibility( - DagTypeConverters.convertToDAGPlan(lr.getVisibility())); + DagTypeConverters.convertToDAGPlan(lr.getVisibility())); if (lr.getType() == LocalResourceType.PATTERN) { if (lr.getPattern() == null || lr.getPattern().isEmpty()) { throw new TezUncheckedException("LocalResource type set to pattern" - + " but pattern is null or empty"); + + " but pattern is null or empty"); } localResourcesBuilder.setPattern(lr.getPattern()); } @@ -582,12 +617,12 @@ public static TezCounters convertTezCountersFromProto(TezCountersProto proto) { TezCounters counters = new TezCounters(); for (TezCounterGroupProto counterGroupProto : proto.getCounterGroupsList()) { CounterGroup group = counters.addGroup(counterGroupProto.getName(), - counterGroupProto.getDisplayName()); + counterGroupProto.getDisplayName()); for (TezCounterProto counterProto : - counterGroupProto.getCountersList()) { + counterGroupProto.getCountersList()) { TezCounter counter = group.findCounter( - counterProto.getName(), - counterProto.getDisplayName()); + counterProto.getName(), + counterProto.getDisplayName()); counter.setValue(counterProto.getValue()); } } @@ -602,7 +637,7 @@ public static TezCountersProto convertTezCountersToProto( while (groupIterator.hasNext()) { CounterGroup counterGroup = groupIterator.next(); TezCounterGroupProto.Builder groupBuilder = - TezCounterGroupProto.newBuilder(); + TezCounterGroupProto.newBuilder(); groupBuilder.setName(counterGroup.getName()); groupBuilder.setDisplayName(counterGroup.getDisplayName()); Iterator counterIterator = counterGroup.iterator(); @@ -610,10 +645,10 @@ public static TezCountersProto convertTezCountersToProto( while (counterIterator.hasNext()) { TezCounter counter = counterIterator.next(); TezCounterProto tezCounterProto = TezCounterProto.newBuilder() - .setName(counter.getName()) - .setDisplayName(counter.getDisplayName()) - .setValue(counter.getValue()) - .build(); + .setName(counter.getName()) + .setDisplayName(counter.getDisplayName()) + .setValue(counter.getValue()) + .build(); groupBuilder.addCounters(counterIndex, tezCounterProto); ++counterIndex; } @@ -626,28 +661,28 @@ public static TezCountersProto convertTezCountersToProto( public static DAGProtos.StatusGetOptsProto convertStatusGetOptsToProto( StatusGetOpts statusGetOpts) { switch (statusGetOpts) { - case GET_COUNTERS: - return DAGProtos.StatusGetOptsProto.GET_COUNTERS; - case GET_MEMORY_USAGE: - return DAGProtos.StatusGetOptsProto.GET_MEMORY_USAGE; + case GET_COUNTERS: + return DAGProtos.StatusGetOptsProto.GET_COUNTERS; + case GET_MEMORY_USAGE: + return DAGProtos.StatusGetOptsProto.GET_MEMORY_USAGE; } throw new TezUncheckedException("Could not convert StatusGetOpts to" + " proto"); } public static StatusGetOpts convertStatusGetOptsFromProto(DAGProtos.StatusGetOptsProto proto) { switch (proto) { - case GET_COUNTERS: - return StatusGetOpts.GET_COUNTERS; - case GET_MEMORY_USAGE: - return StatusGetOpts.GET_MEMORY_USAGE; + case GET_COUNTERS: + return StatusGetOpts.GET_COUNTERS; + case GET_MEMORY_USAGE: + return StatusGetOpts.GET_MEMORY_USAGE; } throw new TezUncheckedException("Could not convert to StatusGetOpts from" + " proto"); } public static List convertStatusGetOptsToProto( - Set statusGetOpts) { + Set statusGetOpts) { List protos = - new ArrayList(statusGetOpts.size()); + new ArrayList(statusGetOpts.size()); for (StatusGetOpts opt : statusGetOpts) { protos.add(convertStatusGetOptsToProto(opt)); } @@ -693,10 +728,10 @@ public static Credentials convertByteStringToCredentials(ByteString byteString) } public static VertexLocationHint convertVertexLocationHintFromProto( - VertexLocationHintProto proto) { + VertexLocationHintProto proto) { List outputList = new ArrayList( - proto.getTaskLocationHintsCount()); - for(PlanTaskLocationHint inputHint : proto.getTaskLocationHintsList()){ + proto.getTaskLocationHintsCount()); + for (PlanTaskLocationHint inputHint : proto.getTaskLocationHintsList()) { TaskLocationHint outputHint = TaskLocationHint.createTaskLocationHint( new HashSet(inputHint.getHostList()), new HashSet(inputHint.getRackList())); @@ -709,12 +744,12 @@ public static VertexLocationHint convertVertexLocationHintFromProto( public static VertexLocationHintProto convertVertexLocationHintToProto( VertexLocationHint vertexLocationHint) { VertexLocationHintProto.Builder builder = - VertexLocationHintProto.newBuilder(); + VertexLocationHintProto.newBuilder(); if (vertexLocationHint.getTaskLocationHints() != null) { for (TaskLocationHint taskLocationHint : - vertexLocationHint.getTaskLocationHints()) { + vertexLocationHint.getTaskLocationHints()) { PlanTaskLocationHint.Builder taskLHBuilder = - PlanTaskLocationHint.newBuilder(); + PlanTaskLocationHint.newBuilder(); if (taskLocationHint.getHosts() != null) { taskLHBuilder.addAllHost(taskLocationHint.getHosts()); } @@ -836,7 +871,6 @@ public static AMPluginDescriptorProto convertServicePluginDescriptorToProto( servicePluginsDescriptor.getTaskCommunicatorDescriptors()); pluginDescriptorBuilder.addAllTaskCommunicators(namedEntityProtos); } - } else { pluginDescriptorBuilder.setContainersEnabled(true).setUberEnabled(false); } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DataSinkDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/DataSinkDescriptor.java index fec2bf6eed..e41c78854c 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/DataSinkDescriptor.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/DataSinkDescriptor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -38,7 +38,7 @@ public class DataSinkDescriptor { private final OutputDescriptor outputDescriptor; private final OutputCommitterDescriptor committerDescriptor; - + private final Credentials credentials; private final Collection urisForCredentials = Sets.newHashSet(); @@ -61,8 +61,8 @@ public class DataSinkDescriptor { */ @Deprecated public DataSinkDescriptor(OutputDescriptor outputDescriptor, - @Nullable OutputCommitterDescriptor committerDescriptor, - @Nullable Credentials credentials) { + @Nullable OutputCommitterDescriptor committerDescriptor, + @Nullable Credentials credentials) { this.outputDescriptor = outputDescriptor; this.committerDescriptor = committerDescriptor; this.credentials = credentials; @@ -86,11 +86,11 @@ public DataSinkDescriptor(OutputDescriptor outputDescriptor, * @param credentials Credentials needs to access the data sink */ public static DataSinkDescriptor create(OutputDescriptor outputDescriptor, - @Nullable OutputCommitterDescriptor committerDescriptor, - @Nullable Credentials credentials) { + @Nullable OutputCommitterDescriptor committerDescriptor, + @Nullable Credentials credentials) { return new DataSinkDescriptor(outputDescriptor, committerDescriptor, credentials); } - + /** * Get the {@link OutputDescriptor} for this {@link DataSinkDescriptor} * @return {@link OutputDescriptor} @@ -98,7 +98,7 @@ public static DataSinkDescriptor create(OutputDescriptor outputDescriptor, public OutputDescriptor getOutputDescriptor() { return outputDescriptor; } - + /** * Get the {@link OutputCommitterDescriptor} for this {@link DataSinkDescriptor} * @return {@link OutputCommitterDescriptor} @@ -106,26 +106,26 @@ public OutputDescriptor getOutputDescriptor() { public @Nullable OutputCommitterDescriptor getOutputCommitterDescriptor() { return committerDescriptor; } - - /** - * This method can be used to specify a list of URIs for which Credentials - * need to be obtained so that the job can run. An incremental list of URIs - * can be provided by making multiple calls to the method. - * - * Currently, @{link credentials} can only be fetched for HDFS and other - * {@link org.apache.hadoop.fs.FileSystem} implementations that support - * credentials. - * - * @param uris - * a list of {@link URI}s - * @return this - */ + + /** + * This method can be used to specify a list of URIs for which Credentials + * need to be obtained so that the job can run. An incremental list of URIs + * can be provided by making multiple calls to the method. + * + * Currently, @{link credentials} can only be fetched for HDFS and other + * {@link org.apache.hadoop.fs.FileSystem} implementations that support + * credentials. + * + * @param uris + * a list of {@link URI}s + * @return this + */ public synchronized DataSinkDescriptor addURIsForCredentials(Collection uris) { Objects.requireNonNull(uris, "URIs cannot be null"); urisForCredentials.addAll(uris); return this; } - + /** * Get the URIs for which credentials will be obtained * @return an unmodifiable list representing the URIs for which credentials @@ -142,5 +142,4 @@ public Collection getURIsForCredentials() { public @Nullable Credentials getCredentials() { return credentials; } - } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DataSourceDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/DataSourceDescriptor.java index db43c91f7a..6838682407 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/DataSourceDescriptor.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/DataSourceDescriptor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -45,7 +45,7 @@ public class DataSourceDescriptor { private final InputDescriptor inputDescriptor; private final InputInitializerDescriptor initializerDescriptor; - + private final Credentials credentials; private final int numShards; private final VertexLocationHint locationHint; @@ -128,13 +128,13 @@ public static DataSourceDescriptor create(InputDescriptor inputDescriptor, } /** - * Get the {@link InputDescriptor} for this {@link DataSourceDescriptor} + * Get the {@link InputDescriptor} for this {@link DataSourceDescriptor} * @return {@link InputDescriptor} */ public InputDescriptor getInputDescriptor() { return inputDescriptor; } - + /** * Get the {@link InputInitializerDescriptor} for this {@link DataSourceDescriptor} * @return {@link InputInitializerDescriptor} @@ -142,26 +142,26 @@ public InputDescriptor getInputDescriptor() { public @Nullable InputInitializerDescriptor getInputInitializerDescriptor() { return initializerDescriptor; } - - /** - * This method can be used to specify a list of URIs for which Credentials - * need to be obtained so that the job can run. An incremental list of URIs - * can be provided by making multiple calls to the method. - * - * Currently, @{link credentials} can only be fetched for HDFS and other - * {@link org.apache.hadoop.fs.FileSystem} implementations that support - * credentials. - * - * @param uris - * a list of {@link URI}s - * @return this - */ + + /** + * This method can be used to specify a list of URIs for which Credentials + * need to be obtained so that the job can run. An incremental list of URIs + * can be provided by making multiple calls to the method. + * + * Currently, @{link credentials} can only be fetched for HDFS and other + * {@link org.apache.hadoop.fs.FileSystem} implementations that support + * credentials. + * + * @param uris + * a list of {@link URI}s + * @return this + */ public synchronized DataSourceDescriptor addURIsForCredentials(Collection uris) { Objects.requireNonNull(uris, "URIs cannot be null"); urisForCredentials.addAll(uris); return this; } - + /** * Get the URIs for which credentials will be obtained * @return an unmodifiable list representing the URIs for which credentials @@ -170,7 +170,7 @@ public synchronized DataSourceDescriptor addURIsForCredentials(Collection u public Collection getURIsForCredentials() { return Collections.unmodifiableCollection(urisForCredentials); } - + /** * Number of shards for this data source. If a vertex has only one * data source this the number of tasks in the vertex should be set to @@ -182,7 +182,7 @@ public Collection getURIsForCredentials() { public int getNumberOfShards() { return numShards; } - + /** * Returns any credentials needed to access this data source. * Is null when this calculation happens on the AppMaster (default) @@ -192,7 +192,7 @@ public int getNumberOfShards() { public @Nullable Credentials getCredentials() { return credentials; } - + /** * Get the location hints for the tasks in the vertex for this data source. * Is null when shard calculation happens on the AppMaster (default) @@ -205,12 +205,10 @@ public int getNumberOfShards() { /** * Get the list of additional local files which were specified during creation. - * @return Map of additional local files or null if there are none + * @return Map of additional local files or null if there are none */ @InterfaceAudience.Private public @Nullable Map getAdditionalLocalFiles() { return additionalLocalFiles; } - - } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/Edge.java b/tez-api/src/main/java/org/apache/tez/dag/api/Edge.java index db509f7050..67be34fe19 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/Edge.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/Edge.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,7 +25,7 @@ * @link {@link EdgeProperty} defines the relationship between them. The producer * vertex provides input to the edge and the consumer vertex reads output from the * edge. - * + * */ @Public public class Edge { @@ -42,8 +42,6 @@ private Edge(Vertex inputVertex, this.edgeProperty = edgeProperty; } - - /** * Creates an edge between the specified vertices. * @@ -70,7 +68,7 @@ public Vertex getInputVertex() { /** * The @link {@link Vertex} that reads output from the edge - * @return {@link Vertex} + * @return {@link Vertex} */ public Vertex getOutputVertex() { return outputVertex; @@ -83,7 +81,7 @@ public Vertex getOutputVertex() { public EdgeProperty getEdgeProperty() { return edgeProperty; } - + /* * Used to identify the edge in the configuration */ @@ -92,7 +90,7 @@ public String getId() { // ensure it is unique. return String.valueOf(System.identityHashCode(this)); } - + @Override public String toString() { return inputVertex + " -> " + outputVertex + " (" + edgeProperty + ")"; diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPlugin.java b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPlugin.java index 4e22f634cd..fe44d180fc 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPlugin.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPlugin.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.api; @@ -41,7 +41,7 @@ public abstract class EdgeManagerPlugin { * Create an instance of the EdgeManagerPlugin. Classes extending this to * create a EdgeManagerPlugin, must provide the same constructor so that Tez * can create an instance of the class at runtime. - * + * * @param context * the context within which this EdgeManagerPlugin will run. Includes * information like configuration which the user may have specified @@ -61,7 +61,7 @@ public EdgeManagerPlugin(EdgeManagerPluginContext context) { * @throws Exception */ public abstract void initialize() throws Exception; - + /** * Get the number of physical inputs on the destination task * @param destinationTaskIndex Index of destination task for which number of @@ -79,14 +79,14 @@ public EdgeManagerPlugin(EdgeManagerPluginContext context) { * @throws Exception */ public abstract int getNumSourceTaskPhysicalOutputs(int sourceTaskIndex) throws Exception; - + /** * Return the routing information to inform consumers about the source task * output that is now available. The return map has the routing information. * The event will be routed to every destination task index in the key of the * map. Every physical input in the value for that task key will receive the * input. - * + * * @param event * Data movement event that contains the output information * @param sourceTaskIndex @@ -99,8 +99,8 @@ public EdgeManagerPlugin(EdgeManagerPluginContext context) { * @throws Exception */ public abstract void routeDataMovementEventToDestination(DataMovementEvent event, - int sourceTaskIndex, int sourceOutputIndex, - Map> destinationTaskAndInputIndices) throws Exception; + int sourceTaskIndex, int sourceOutputIndex, + Map> destinationTaskAndInputIndices) throws Exception; /** * Return the routing information to inform consumers about the failure of a @@ -110,7 +110,7 @@ public abstract void routeDataMovementEventToDestination(DataMovementEvent event * for that task key will receive the failure notification. This method will * be called once for every source task failure and information for all * affected destinations must be provided in that invocation. - * + * * @param sourceTaskIndex * Source task * @param destinationTaskAndInputIndices @@ -118,7 +118,7 @@ public abstract void routeDataMovementEventToDestination(DataMovementEvent event * @throws Exception */ public abstract void routeInputSourceTaskFailedEventToDestination(int sourceTaskIndex, - Map> destinationTaskAndInputIndices) throws Exception; + Map> destinationTaskAndInputIndices) throws Exception; /** * Get the number of destination tasks that consume data from the source task @@ -126,10 +126,10 @@ public abstract void routeInputSourceTaskFailedEventToDestination(int sourceTask * @throws Exception */ public abstract int getNumDestinationConsumerTasks(int sourceTaskIndex) throws Exception; - + /** * Return the source task index to which to send the input error event - * + * * @param event * Input read error event. Has more information about the error * @param destinationTaskIndex @@ -141,7 +141,7 @@ public abstract void routeInputSourceTaskFailedEventToDestination(int sourceTask * @throws Exception */ public abstract int routeInputErrorEventToSource(InputReadErrorEvent event, - int destinationTaskIndex, int destinationFailedInputIndex) throws Exception; + int destinationTaskIndex, int destinationFailedInputIndex) throws Exception; /** * Return the {@link org.apache.tez.dag.api.EdgeManagerPluginContext} for this specific instance of diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginContext.java b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginContext.java index ef6925b4ee..f36d10d5d8 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginContext.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginContext.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,19 +29,19 @@ @Public @Unstable public interface EdgeManagerPluginContext { - + /** * Returns the payload specified by the user for the edge. * @return the {@link org.apache.tez.dag.api.UserPayload} specified by the user */ public UserPayload getUserPayload(); - + /** * Returns the source vertex name * @return the source vertex name */ public String getSourceVertexName(); - + /** * Returns the destination vertex name * @return the destination vertex name diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginDescriptor.java index b2d2de6d83..49546140f3 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginDescriptor.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginDescriptor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginOnDemand.java b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginOnDemand.java index 3d7f2ab2b7..7fd4267984 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginOnDemand.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginOnDemand.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.api; @@ -77,7 +77,7 @@ public static class EventRouteMetadata { private final int numEvents; private final int[] targetIndices; private final int[] sourceIndices; - + /** * Create an {@link EventRouteMetadata} that will create numEvents copies of * the {@link Event} to be routed. Use this to create @@ -85,7 +85,7 @@ public static class EventRouteMetadata { * {@link InputFailedEvent}s where the target input indices must be * specified to route those events. Typically numEvents would be 1 for these * events. - * + * * @param numEvents * Number of copies of the event to be routed * @param targetIndices @@ -97,14 +97,14 @@ public static class EventRouteMetadata { public static EventRouteMetadata create(int numEvents, int[] targetIndices) { return new EventRouteMetadata(numEvents, targetIndices, null); } - + /** * Create an {@link EventRouteMetadata} that will create numEvents copies of * the {@link Event} to be routed. Use this to create * {@link EventRouteMetadata} for {@link CompositeDataMovementEvent} where * the target input indices and source output indices must be specified to * route those events. Typically numEvents would be 1 for these events. - * + * * @param numEvents * Number of copies of the event to be routed * @param targetIndices @@ -157,7 +157,7 @@ public int getNumEvents() { * extending this to create a {@link EdgeManagerPluginOnDemand}, must provide * the same constructor so that Tez can create an instance of the class at * runtime. - * + * * @param context * the context within which this {@link EdgeManagerPluginOnDemand} * will run. Includes information like configuration which the user @@ -177,7 +177,7 @@ public EdgeManagerPluginOnDemand(EdgeManagerPluginContext context) { * @throws Exception */ public abstract void initialize() throws Exception; - + /** * This method will be invoked just before routing of events will begin. The * plugin can use this opportunity to make any runtime initialization's that @@ -202,17 +202,17 @@ public EdgeManagerPluginOnDemand(EdgeManagerPluginContext context) { * @throws Exception */ public abstract int getNumSourceTaskPhysicalOutputs(int sourceTaskIndex) throws Exception; - + /** * Get the number of destination tasks that consume data from the source task * @param sourceTaskIndex Source task index * @throws Exception */ public abstract int getNumDestinationConsumerTasks(int sourceTaskIndex) throws Exception; - + /** * Return the source task index to which to send the input error event - * + * * @param destinationTaskIndex * Destination task that reported the error * @param destinationFailedInputIndex @@ -222,8 +222,8 @@ public EdgeManagerPluginOnDemand(EdgeManagerPluginContext context) { * @throws Exception */ public abstract int routeInputErrorEventToSource(int destinationTaskIndex, - int destinationFailedInputIndex) throws Exception; - + int destinationFailedInputIndex) throws Exception; + /** * The method provides the {@link EventRouteMetadata} to route a * {@link DataMovementEvent} produced by the given source task to the given @@ -231,7 +231,7 @@ public abstract int routeInputErrorEventToSource(int destinationTaskIndex, * target input indices set to enable the routing. If the routing metadata is * common across different events then the plugin can cache and reuse the same * object. - * + * * @param sourceTaskIndex * The index of the task in the source vertex of this edge that * produced a {@link DataMovementEvent} @@ -245,7 +245,7 @@ public abstract int routeInputErrorEventToSource(int destinationTaskIndex, * @throws Exception */ public abstract @Nullable EventRouteMetadata routeDataMovementEventToDestination(int sourceTaskIndex, - int sourceOutputIndex, int destinationTaskIndex) throws Exception; + int sourceOutputIndex, int destinationTaskIndex) throws Exception; /** * The method provides the {@link EventRouteMetadata} to route a @@ -254,7 +254,7 @@ public abstract int routeInputErrorEventToSource(int destinationTaskIndex, * the target input indices and source output indices set to enable the * routing. If the routing metadata is common across different events then the * plugin can cache and reuse the same object. - * + * * @param sourceTaskIndex * The index of the task in the source vertex of this edge that * produced a {@link CompositeDataMovementEvent} @@ -275,7 +275,7 @@ public abstract int routeInputErrorEventToSource(int destinationTaskIndex, * target input indices set to enable the routing. If the routing metadata is * common across different events then the plugin can cache and reuse the same * object. - * + * * @param sourceTaskIndex * The index of the failed task in the source vertex of this edge. * @param destinationTaskIndex @@ -287,7 +287,7 @@ public abstract int routeInputErrorEventToSource(int destinationTaskIndex, */ public abstract @Nullable EventRouteMetadata routeInputSourceTaskFailedEventToDestination( int sourceTaskIndex, int destinationTaskIndex) throws Exception; - + /** * Return the {@link org.apache.tez.dag.api.EdgeManagerPluginContext} for this specific instance of * the vertex manager. @@ -299,13 +299,14 @@ public EdgeManagerPluginContext getContext() { } // Empty implementations of EdgeManagerPlugin interfaces that are not needed + /** * Return the routing information to inform consumers about the source task * output that is now available. The return map has the routing information. * The event will be routed to every destination task index in the key of the * map. Every physical input in the value for that task key will receive the * input. - * + * * @param event * Data movement event that contains the output information * @param sourceTaskIndex @@ -318,8 +319,8 @@ public EdgeManagerPluginContext getContext() { * @throws Exception */ public void routeDataMovementEventToDestination(DataMovementEvent event, - int sourceTaskIndex, int sourceOutputIndex, - Map> destinationTaskAndInputIndices) throws Exception {} + int sourceTaskIndex, int sourceOutputIndex, + Map> destinationTaskAndInputIndices) throws Exception {} /** * Return the routing information to inform consumers about the failure of a @@ -329,7 +330,7 @@ public void routeDataMovementEventToDestination(DataMovementEvent event, * for that task key will receive the failure notification. This method will * be called once for every source task failure and information for all * affected destinations must be provided in that invocation. - * + * * @param sourceTaskIndex * Source task * @param destinationTaskAndInputIndices @@ -337,11 +338,11 @@ public void routeDataMovementEventToDestination(DataMovementEvent event, * @throws Exception */ public void routeInputSourceTaskFailedEventToDestination(int sourceTaskIndex, - Map> destinationTaskAndInputIndices) throws Exception {} + Map> destinationTaskAndInputIndices) throws Exception {} /** * Return the source task index to which to send the input error event - * + * * @param event * Input read error event. Has more information about the error * @param destinationTaskIndex @@ -353,8 +354,7 @@ public void routeInputSourceTaskFailedEventToDestination(int sourceTaskIndex, * @throws Exception */ public int routeInputErrorEventToSource(InputReadErrorEvent event, - int destinationTaskIndex, int destinationFailedInputIndex) throws Exception { + int destinationTaskIndex, int destinationFailedInputIndex) throws Exception { return routeInputErrorEventToSource(destinationTaskIndex, destinationFailedInputIndex); } - } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java index 1850060ece..e53c76d1d1 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,7 +34,6 @@ @Public public class EdgeProperty { - /** * Defines the manner of data movement between source and destination tasks. * Determines which destination tasks have access to data produced on this @@ -126,7 +125,6 @@ public enum ConcurrentEdgeTriggerType { SOURCE_TASK_STARTED } - final DataMovementType dataMovementType; final DataSourceType dataSourceType; final SchedulingType schedulingType; @@ -177,8 +175,8 @@ public static EdgeProperty create(EdgeManagerPluginDescriptor edgeManagerDescrip @Private public static EdgeProperty create(EdgeManagerPluginDescriptor edgeManagerDescriptor, - DataMovementType dataMovementType, DataSourceType dataSourceType, - SchedulingType schedulingType, OutputDescriptor edgeSource, InputDescriptor edgeDestination) { + DataMovementType dataMovementType, DataSourceType dataSourceType, + SchedulingType schedulingType, OutputDescriptor edgeSource, InputDescriptor edgeDestination) { return new EdgeProperty(edgeManagerDescriptor, dataMovementType, dataSourceType, schedulingType, edgeSource, edgeDestination); } @@ -193,7 +191,6 @@ private EdgeProperty(DataMovementType dataMovementType, DataMovementType.CUSTOM + " cannot be used with this constructor"); } - private EdgeProperty(EdgeManagerPluginDescriptor edgeManagerDescriptor, DataSourceType dataSourceType, SchedulingType schedulingType, @@ -204,8 +201,8 @@ private EdgeProperty(EdgeManagerPluginDescriptor edgeManagerDescriptor, } private EdgeProperty(EdgeManagerPluginDescriptor edgeManagerDescriptor, - DataMovementType dataMovementType, DataSourceType dataSourceType, - SchedulingType schedulingType, OutputDescriptor edgeSource, InputDescriptor edgeDestination) { + DataMovementType dataMovementType, DataSourceType dataSourceType, + SchedulingType schedulingType, OutputDescriptor edgeSource, InputDescriptor edgeDestination) { this.dataMovementType = dataMovementType; this.edgeManagerDescriptor = edgeManagerDescriptor; this.dataSourceType = dataSourceType; @@ -268,5 +265,4 @@ public String toString() { + " >> " + (edgeManagerDescriptor == null ? "NullEdgeManager" : edgeManagerDescriptor.getClassName()) + " }"; } - } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/EntityDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/EntityDescriptor.java index 2c7b834c1c..5113a835b2 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/EntityDescriptor.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/EntityDescriptor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -35,8 +35,8 @@ * Describes a given user code entity. Consists of the name of the class implementing * the user logic and a payload that can be used to configure an object instance of * that class. In addition some history information can be set for logging/debugging. - *
This is not supposed to be extended by users. Users are expected to use the derived - * classes for specific entities + *
This is not supposed to be extended by users. Users are expected to use the derived + * classes for specific entities */ @Public @SuppressWarnings("unchecked") @@ -137,7 +137,7 @@ public void readFields(DataInput in) throws IOException { if (payloadLength != -1) { byte[] bb = new byte[payloadLength]; in.readFully(bb); - int version =in.readInt(); + int version = in.readInt(); this.userPayload = DagTypeConverters.convertToTezUserPayload(ByteBuffer.wrap(bb), version); } } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/GroupInputEdge.java b/tez-api/src/main/java/org/apache/tez/dag/api/GroupInputEdge.java index 9eb9a1a935..72a9bbd04d 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/GroupInputEdge.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/GroupInputEdge.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -88,7 +88,7 @@ public Vertex getOutputVertex() { public EdgeProperty getEdgeProperty() { return edgeProperty; } - + InputDescriptor getMergedInput() { return mergedInput; } @@ -100,7 +100,7 @@ InputDescriptor getMergedInput() { public String getId() { return String.valueOf(this.hashCode()); } - + @Override public String toString() { return inputVertexGroup + " -> " + outputVertex + " (" + edgeProperty + ")"; diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/HistoryLogLevel.java b/tez-api/src/main/java/org/apache/tez/dag/api/HistoryLogLevel.java index 96d74f965f..de1b6fa687 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/HistoryLogLevel.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/HistoryLogLevel.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/InputDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/InputDescriptor.java index 012c83e954..06e4d38238 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/InputDescriptor.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/InputDescriptor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/InputInitializerDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/InputInitializerDescriptor.java index bb2be5d84f..a6be2d8381 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/InputInitializerDescriptor.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/InputInitializerDescriptor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/NamedEntityDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/NamedEntityDescriptor.java index 86ae26c876..23a47ccae1 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/NamedEntityDescriptor.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/NamedEntityDescriptor.java @@ -22,7 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience; @SuppressWarnings("unchecked") -public class NamedEntityDescriptor> extends EntityDescriptor> { +public class NamedEntityDescriptor> extends EntityDescriptor> { private final String entityName; @InterfaceAudience.Private diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/OutputCommitterDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/OutputCommitterDescriptor.java index fd8c86d402..22d9db6636 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/OutputCommitterDescriptor.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/OutputCommitterDescriptor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/OutputDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/OutputDescriptor.java index 58b6ce4630..9671d7cf62 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/OutputDescriptor.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/OutputDescriptor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -32,7 +32,7 @@ public class OutputDescriptor extends EntityDescriptor { public OutputDescriptor() { super(); } - + private OutputDescriptor(String outputClassName) { super(outputClassName); } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/PreWarmVertex.java b/tez-api/src/main/java/org/apache/tez/dag/api/PreWarmVertex.java index 62979b5088..a9e50529b7 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/PreWarmVertex.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/PreWarmVertex.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -53,18 +53,16 @@ @Public public class PreWarmVertex extends Vertex { - private PreWarmVertex(String vertexName, ProcessorDescriptor processorDescriptor, int parallelism, Resource taskResource) { super(vertexName, processorDescriptor, parallelism, taskResource); } - private PreWarmVertex(String vertexName, int parallelism, Resource taskResource) { this(vertexName, ProcessorDescriptor.create( "org.apache.tez.runtime.library.processor.PreWarmProcessor"), parallelism, taskResource); } - + /** * Create a config builder for the @link {@link PreWarmVertex}. This may be used to construct the * pre-warm vertex more flexibly. @@ -128,31 +126,31 @@ public static class PreWarmVertexConfigBuilder { ProcessorDescriptor proc; Resource resource; Configuration conf; - + PreWarmVertexConfigBuilder(Configuration conf) { this.conf = conf; } - + public PreWarmVertexConfigBuilder setName(String name) { this.name = name; return this; } - + public PreWarmVertexConfigBuilder setProcessorDescriptor(ProcessorDescriptor proc) { this.proc = proc; return this; } - + public PreWarmVertexConfigBuilder setResource(Resource resource) { this.resource = resource; return this; } - + public PreWarmVertexConfigBuilder setParallelism(int parallelism) { this.parallelism = parallelism; return this; } - + public PreWarmVertex build() { if (name == null) { name = "_PreWarm_"; @@ -160,7 +158,7 @@ public PreWarmVertex build() { if (parallelism == 0) { parallelism = conf.getInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, -1); if (parallelism == -1) { - throw new TezUncheckedException("Prewarm parallelism must be set or specified in conf via " + throw new TezUncheckedException("Prewarm parallelism must be set or specified in conf via " + TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS); } } @@ -168,9 +166,8 @@ public PreWarmVertex build() { proc = ProcessorDescriptor.create("org.apache.tez.runtime.library.processor.PreWarmProcessor"); } - + return create(name, proc, parallelism, resource); } } - } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/ProcessorDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/ProcessorDescriptor.java index 7f088d8caf..a3645904f1 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/ProcessorDescriptor.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/ProcessorDescriptor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -32,7 +32,7 @@ public class ProcessorDescriptor extends EntityDescriptor { public ProcessorDescriptor() { super(); } - + private ProcessorDescriptor(String processorClassName) { super(processorClassName); } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/RootInputLeafOutput.java b/tez-api/src/main/java/org/apache/tez/dag/api/RootInputLeafOutput.java index 953a78da6e..c8a18a3d73 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/RootInputLeafOutput.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/RootInputLeafOutput.java @@ -1,27 +1,27 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.api; import org.apache.hadoop.classification.InterfaceAudience.Private; @Private -public class RootInputLeafOutput , S extends EntityDescriptor> { +public class RootInputLeafOutput, S extends EntityDescriptor> { private final String name; private final T ioDescriptor; @@ -32,7 +32,7 @@ public RootInputLeafOutput(String name, T ioDescriptor, S controllerDescriptor) this.ioDescriptor = ioDescriptor; this.controllerDescriptor = controllerDescriptor; } - + public String getName() { return this.name; } @@ -40,7 +40,7 @@ public String getName() { public T getIODescriptor() { return this.ioDescriptor; } - + public S getControllerDescriptor() { return this.controllerDescriptor; } @@ -50,5 +50,4 @@ public String toString() { return "{InputName=" + name + "}, {Descriptor=" + ioDescriptor + "}, {ControllerDescriptor=" + controllerDescriptor + "}"; } - } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/Scope.java b/tez-api/src/main/java/org/apache/tez/dag/api/Scope.java index f638e092a5..99c314bbc0 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/Scope.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/Scope.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/SessionNotReady.java b/tez-api/src/main/java/org/apache/tez/dag/api/SessionNotReady.java index 8c06a4d1d5..fcecd112bd 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/SessionNotReady.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/SessionNotReady.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/SessionNotRunning.java b/tez-api/src/main/java/org/apache/tez/dag/api/SessionNotRunning.java index 22544d53eb..1697329fa5 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/SessionNotRunning.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/SessionNotRunning.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -40,5 +40,4 @@ public SessionNotRunning(String message) { public SessionNotRunning(Throwable cause) { super(cause); } - } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TaskLocationHint.java b/tez-api/src/main/java/org/apache/tez/dag/api/TaskLocationHint.java index 3070c338cc..354eede4c0 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TaskLocationHint.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TaskLocationHint.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -44,16 +44,20 @@ public class TaskLocationHint { public static class TaskBasedLocationAffinity { private String vertexName; private int taskIndex; + public TaskBasedLocationAffinity(String vertexName, int taskIndex) { this.vertexName = vertexName; this.taskIndex = taskIndex; } + public String getVertexName() { return vertexName; } + public int getTaskIndex() { return taskIndex; } + @Override public String toString() { return "[Vertex: " + vertexName + ", TaskIndex: " + taskIndex + "]"; @@ -64,7 +68,7 @@ public String toString() { private Set hosts; // Rack names if any to be used private Set racks; - + private TaskBasedLocationAffinity affinitizedTask; private TaskLocationHint(String vertexName, int taskIndex) { @@ -124,10 +128,10 @@ public Set getRacks() { public int hashCode() { final int prime = 9397; int result = 1; - result = ( hosts != null) ? + result = (hosts != null) ? prime * result + hosts.hashCode() : result + prime; - result = ( racks != null) ? + result = (racks != null) ? prime * result + racks.hashCode() : result + prime; if (affinitizedTask != null) { @@ -171,7 +175,7 @@ public boolean equals(Object obj) { } else if (!affinitizedTask.getVertexName().equals(other.affinitizedTask.getVertexName())) { return false; } - } else if (other.affinitizedTask != null){ + } else if (other.affinitizedTask != null) { return false; } return true; diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java index 1ffd70a3ff..f8cbd244fd 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -27,6 +27,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; + import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -42,7 +43,6 @@ import com.google.common.annotations.VisibleForTesting; - /** * Defines the configurations for Tez. These configurations are typically specified in * tez-site.xml on the client machine where TezClient is used to launch the Tez application. @@ -87,7 +87,6 @@ public class TezConfiguration extends Configuration { TezConfiguration.TEZ_TASK_MAX_EVENTS_PER_HEARTBEAT); setupConfigurationScope(TezConfiguration.class); - } static void setupConfigurationScope(Class clazz) { @@ -175,7 +174,7 @@ public TezConfiguration(boolean loadDefaults) { * to all task credentials. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_AM_CREDENTIALS_MERGE = TEZ_AM_PREFIX + "credentials-merge"; public static boolean TEZ_AM_CREDENTIALS_MERGE_DEFAULT = true; @@ -214,7 +213,7 @@ public TezConfiguration(boolean loadDefaults) { * mode is recommended. If session mode is enabled then container reuse is recommended. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_AM_SESSION_MODE = TEZ_AM_PREFIX + "mode.session"; public static final boolean TEZ_AM_SESSION_MODE_DEFAULT = false; @@ -295,7 +294,7 @@ public TezConfiguration(boolean loadDefaults) { * Expert level setting. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_AM_MAX_ALLOWED_TIME_FOR_TASK_READ_ERROR_SEC = TEZ_AM_PREFIX + "max.allowed.time-sec.for-read-error"; public static final int TEZ_AM_MAX_ALLOWED_TIME_FOR_TASK_READ_ERROR_SEC_DEFAULT = 300; @@ -313,7 +312,7 @@ public TezConfiguration(boolean loadDefaults) { * Expert level setting. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_AM_MAX_ALLOWED_DOWNSTREAM_HOST_FAILURES_FRACTION = TEZ_AM_PREFIX + "max.allowed.downstream.host.failures.fraction"; public static final double TEZ_AM_MAX_ALLOWED_DOWNSTREAM_HOST_FAILURES_FRACTION_DEFAULT = 0.2; @@ -327,7 +326,7 @@ public TezConfiguration(boolean loadDefaults) { * this value must be appropriately chosen. Defaults to the safe choice of true. */ @ConfigurationScope(Scope.DAG) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS = TEZ_AM_PREFIX + "commit-all-outputs-on-dag-success"; public static final boolean TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS_DEFAULT = true; @@ -361,7 +360,7 @@ public TezConfiguration(boolean loadDefaults) { * */ @ConfigurationScope(Scope.AM) @ConfigurationProperty - public static final String TEZ_AM_LAUNCH_CMD_OPTS = TEZ_AM_PREFIX + "launch.cmd-opts"; + public static final String TEZ_AM_LAUNCH_CMD_OPTS = TEZ_AM_PREFIX + "launch.cmd-opts"; public static final String TEZ_AM_LAUNCH_CMD_OPTS_JDK8_DEFAULT = "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC"; public static final String TEZ_AM_LAUNCH_CMD_OPTS_JDK9_DEFAULT = @@ -420,14 +419,14 @@ public TezConfiguration(boolean loadDefaults) { * smaller than 4GB and use 0.8 for larger container. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="float") + @ConfigurationProperty(type = "float") public static final String TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION = TEZ_PREFIX + "container.max.java.heap.fraction"; public static final double TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_DEFAULT = 0.8; private static final String NATIVE_LIB_PARAM_DEFAULT = Shell.WINDOWS ? - "PATH=%PATH%;%HADOOP_COMMON_HOME%\\bin": - "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HADOOP_COMMON_HOME/lib/native/"; + "PATH=%PATH%;%HADOOP_COMMON_HOME%\\bin" : + "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HADOOP_COMMON_HOME/lib/native/"; /** * String value. Env settings will be merged with {@link #TEZ_AM_LAUNCH_ENV} @@ -468,7 +467,6 @@ public TezConfiguration(boolean loadDefaults) { public static final String TEZ_CONCURRENT_EDGE_TRIGGER_TYPE_DEFAULT = ConcurrentEdgeTriggerType.SOURCE_VERTEX_CONFIGURED.name(); - /** * String value. Env settings will be merged with {@link #TEZ_TASK_LAUNCH_ENV} * during the launch of the task process. This property will typically be configured to @@ -498,7 +496,7 @@ public TezConfiguration(boolean loadDefaults) { @Private @ConfigurationScope(Scope.DAG) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_CANCEL_DELEGATION_TOKENS_ON_COMPLETION = TEZ_PREFIX + "cancel.delegation.tokens.on.completion"; public static final boolean TEZ_CANCEL_DELEGATION_TOKENS_ON_COMPLETION_DEFAULT = true; @@ -515,7 +513,7 @@ public TezConfiguration(boolean loadDefaults) { * Expert level setting. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_AM_TASK_LISTENER_THREAD_COUNT = TEZ_AM_PREFIX + "task.listener.thread-count"; public static final int TEZ_AM_TASK_LISTENER_THREAD_COUNT_DEFAULT = 30; @@ -528,7 +526,7 @@ public TezConfiguration(boolean loadDefaults) { */ @Unstable @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_COUNTERS_MAX = TEZ_PREFIX + "counters.max"; public static final int TEZ_COUNTERS_MAX_DEFAULT = 1200; @@ -539,7 +537,7 @@ public TezConfiguration(boolean loadDefaults) { */ @Unstable @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_COUNTERS_MAX_GROUPS = TEZ_PREFIX + "counters.max.groups"; public static final int TEZ_COUNTERS_MAX_GROUPS_DEFAULT = 500; @@ -550,7 +548,7 @@ public TezConfiguration(boolean loadDefaults) { */ @Unstable @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_COUNTERS_COUNTER_NAME_MAX_LENGTH = TEZ_PREFIX + "counters.counter-name.max-length"; public static final int TEZ_COUNTERS_COUNTER_NAME_MAX_LENGTH_DEFAULT = 64; @@ -562,7 +560,7 @@ public TezConfiguration(boolean loadDefaults) { */ @Unstable @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH = TEZ_PREFIX + "counters.group-name.max-length"; public static final int TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH_DEFAULT = 256; @@ -573,7 +571,7 @@ public TezConfiguration(boolean loadDefaults) { * when some tasks are running slower due bad/slow machines */ @ConfigurationScope(Scope.VERTEX) // TODO Verify the vertex speculation, TEZ-1788 - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_AM_SPECULATION_ENABLED = TEZ_AM_PREFIX + "speculation.enabled"; public static final boolean TEZ_AM_SPECULATION_ENABLED_DEFAULT = false; @@ -583,9 +581,9 @@ public TezConfiguration(boolean loadDefaults) { */ @Unstable @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="float") + @ConfigurationProperty(type = "float") public static final String TEZ_AM_LEGACY_SPECULATIVE_SLOWTASK_THRESHOLD = - TEZ_AM_PREFIX + "legacy.speculative.slowtask.threshold"; + TEZ_AM_PREFIX + "legacy.speculative.slowtask.threshold"; /** * Long value. Specifies the timeout after which tasks on a single task vertex must be speculated. @@ -593,9 +591,9 @@ public TezConfiguration(boolean loadDefaults) { */ @Unstable @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="long") + @ConfigurationProperty(type = "long") public static final String TEZ_AM_LEGACY_SPECULATIVE_SINGLE_TASK_VERTEX_TIMEOUT = - TEZ_AM_PREFIX + "legacy.speculative.single.task.vertex.timeout"; + TEZ_AM_PREFIX + "legacy.speculative.single.task.vertex.timeout"; public static final long TEZ_AM_LEGACY_SPECULATIVE_SINGLE_TASK_VERTEX_TIMEOUT_DEFAULT = -1; @Private @@ -608,9 +606,9 @@ public TezConfiguration(boolean loadDefaults) { */ @Unstable @ConfigurationScope(Scope.VERTEX) - @ConfigurationProperty(type="long") + @ConfigurationProperty(type = "long") public static final String TEZ_AM_SOONEST_RETRY_AFTER_NO_SPECULATE = - TEZ_AM_PREFIX + "soonest.retry.after.no.speculate"; + TEZ_AM_PREFIX + "soonest.retry.after.no.speculate"; public static final long TEZ_AM_SOONEST_RETRY_AFTER_NO_SPECULATE_DEFAULT = 1000L * 1L; /** @@ -619,9 +617,9 @@ public TezConfiguration(boolean loadDefaults) { */ @Unstable @ConfigurationScope(Scope.VERTEX) - @ConfigurationProperty(type="long") - public static final String TEZ_AM_SOONEST_RETRY_AFTER_SPECULATE= - TEZ_AM_PREFIX + "soonest.retry.after.speculate"; + @ConfigurationProperty(type = "long") + public static final String TEZ_AM_SOONEST_RETRY_AFTER_SPECULATE = + TEZ_AM_PREFIX + "soonest.retry.after.speculate"; public static final long TEZ_AM_SOONEST_RETRY_AFTER_SPECULATE_DEFAULT = 1000L * 15L; /** The class that should be used for speculative execution calculations. */ @@ -640,7 +638,7 @@ public TezConfiguration(boolean loadDefaults) { */ @Unstable @ConfigurationScope(Scope.VERTEX) - @ConfigurationProperty(type="long") + @ConfigurationProperty(type = "long") public static final String TEZ_AM_ESTIMATOR_EXPONENTIAL_LAMBDA_MS = TEZ_ESTIMATOR_PREFIX + "exponential.lambda.ms"; public static final long TEZ_AM_ESTIMATOR_EXPONENTIAL_LAMBDA_MS_DEFAULT = @@ -652,7 +650,7 @@ public TezConfiguration(boolean loadDefaults) { */ @Unstable @ConfigurationScope(Scope.VERTEX) - @ConfigurationProperty(type="long") + @ConfigurationProperty(type = "long") public static final String TEZ_AM_ESTIMATOR_EXPONENTIAL_STAGNATED_MS = TEZ_ESTIMATOR_PREFIX + "exponential.stagnated.ms"; public static final long TEZ_AM_ESTIMATOR_EXPONENTIAL_STAGNATED_MS_DEFAULT = @@ -665,7 +663,7 @@ public TezConfiguration(boolean loadDefaults) { */ @Unstable @ConfigurationScope(Scope.VERTEX) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_AM_ESTIMATOR_EXPONENTIAL_SKIP_INITIALS = TEZ_ESTIMATOR_PREFIX + "exponential.skip.initials"; public static final int TEZ_AM_ESTIMATOR_EXPONENTIAL_SKIP_INITIALS_DEFAULT = 24; @@ -675,9 +673,9 @@ public TezConfiguration(boolean loadDefaults) { */ @Unstable @ConfigurationScope(Scope.VERTEX) - @ConfigurationProperty(type="double") + @ConfigurationProperty(type = "double") public static final String TEZ_AM_PROPORTION_RUNNING_TASKS_SPECULATABLE = - TEZ_AM_PREFIX + "proportion.running.tasks.speculatable"; + TEZ_AM_PREFIX + "proportion.running.tasks.speculatable"; public static final double TEZ_AM_PROPORTION_RUNNING_TASKS_SPECULATABLE_DEFAULT = 0.1; /** @@ -685,9 +683,9 @@ public TezConfiguration(boolean loadDefaults) { */ @Unstable @ConfigurationScope(Scope.VERTEX) - @ConfigurationProperty(type="double") + @ConfigurationProperty(type = "double") public static final String TEZ_AM_PROPORTION_TOTAL_TASKS_SPECULATABLE = - TEZ_AM_PREFIX + "proportion.total.tasks.speculatable"; + TEZ_AM_PREFIX + "proportion.total.tasks.speculatable"; public static final double TEZ_AM_PROPORTION_TOTAL_TASKS_SPECULATABLE_DEFAULT = 0.01; /** @@ -695,9 +693,9 @@ public TezConfiguration(boolean loadDefaults) { */ @Unstable @ConfigurationScope(Scope.VERTEX) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_AM_MINIMUM_ALLOWED_SPECULATIVE_TASKS = - TEZ_AM_PREFIX + "minimum.allowed.speculative.tasks"; + TEZ_AM_PREFIX + "minimum.allowed.speculative.tasks"; public static final int TEZ_AM_MINIMUM_ALLOWED_SPECULATIVE_TASKS_DEFAULT = 10; /** @@ -705,19 +703,18 @@ public TezConfiguration(boolean loadDefaults) { * master. Expert level setting. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT = - TEZ_AM_PREFIX + "containerlauncher.thread-count-limit"; + TEZ_AM_PREFIX + "containerlauncher.thread-count-limit"; public static final int TEZ_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT_DEFAULT = - 500; - + 500; /** * Int value. Specifies the number of task failures on a node before the node is considered faulty. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_AM_MAX_TASK_FAILURES_PER_NODE = TEZ_AM_PREFIX + "maxtaskfailures.per.node"; public static final int TEZ_AM_MAX_TASK_FAILURES_PER_NODE_DEFAULT = 10; @@ -729,7 +726,7 @@ public TezConfiguration(boolean loadDefaults) { * Expert level setting. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_AM_MAX_APP_ATTEMPTS = TEZ_AM_PREFIX + "max.app.attempts"; public static final int TEZ_AM_MAX_APP_ATTEMPTS_DEFAULT = 2; @@ -739,7 +736,7 @@ public TezConfiguration(boolean loadDefaults) { * Setting <=0 implies no limit */ @ConfigurationScope(Scope.VERTEX) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_AM_VERTEX_MAX_TASK_CONCURRENCY = TEZ_AM_PREFIX + "vertex.max-task-concurrency"; public static final int TEZ_AM_VERTEX_MAX_TASK_CONCURRENCY_DEFAULT = -1; @@ -749,7 +746,7 @@ public TezConfiguration(boolean loadDefaults) { * This does not count killed attempts. Task failure results in DAG failure. */ @ConfigurationScope(Scope.VERTEX) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_AM_TASK_MAX_FAILED_ATTEMPTS = TEZ_AM_PREFIX + "task.max.failed.attempts"; public static final int TEZ_AM_TASK_MAX_FAILED_ATTEMPTS_DEFAULT = 4; @@ -769,27 +766,27 @@ public TezConfiguration(boolean loadDefaults) { * failures gets higher priority */ @ConfigurationScope(Scope.VERTEX) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_AM_TASK_RESCHEDULE_HIGHER_PRIORITY = TEZ_AM_PREFIX + "task.reschedule.higher.priority"; - public static final boolean TEZ_AM_TASK_RESCHEDULE_HIGHER_PRIORITY_DEFAULT=true; + public static final boolean TEZ_AM_TASK_RESCHEDULE_HIGHER_PRIORITY_DEFAULT = true; /** * Boolean value. Specifies whether a re-scheduled attempt of a task, caused by previous * failure get relaxed locality */ @ConfigurationScope(Scope.VERTEX) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_AM_TASK_RESCHEDULE_RELAXED_LOCALITY = TEZ_AM_PREFIX + "task.reschedule.relaxed.locality"; - public static final boolean TEZ_AM_TASK_RESCHEDULE_RELAXED_LOCALITY_DEFAULT=true; + public static final boolean TEZ_AM_TASK_RESCHEDULE_RELAXED_LOCALITY_DEFAULT = true; /** * Boolean value. Enabled blacklisting of nodes of nodes that are considered faulty. These nodes * will not be used to execute tasks. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_AM_NODE_BLACKLISTING_ENABLED = TEZ_AM_PREFIX + "node-blacklisting.enabled"; public static final boolean TEZ_AM_NODE_BLACKLISTING_ENABLED_DEFAULT = true; @@ -800,7 +797,7 @@ public TezConfiguration(boolean loadDefaults) { * temporary surges in failures (e.g. due to network outages). */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_AM_NODE_BLACKLISTING_IGNORE_THRESHOLD = TEZ_AM_PREFIX + "node-blacklisting.ignore-threshold-node-percent"; public static final int TEZ_AM_NODE_BLACKLISTING_IGNORE_THRESHOLD_DEFAULT = 33; @@ -812,15 +809,15 @@ public TezConfiguration(boolean loadDefaults) { * errors from downstream tasks. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_AM_NODE_UNHEALTHY_RESCHEDULE_TASKS = TEZ_AM_PREFIX + "node-unhealthy-reschedule-tasks"; public static final boolean - TEZ_AM_NODE_UNHEALTHY_RESCHEDULE_TASKS_DEFAULT = false; + TEZ_AM_NODE_UNHEALTHY_RESCHEDULE_TASKS_DEFAULT = false; /** Int value. Number of threads to handle client RPC requests. Expert level setting.*/ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_AM_CLIENT_THREAD_COUNT = TEZ_AM_PREFIX + "client.am.thread-count"; public static final int TEZ_AM_CLIENT_THREAD_COUNT_DEFAULT = 2; @@ -864,21 +861,21 @@ public TezConfiguration(boolean loadDefaults) { /** Int value. The amount of memory in MB to be used by the AppMaster */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_AM_RESOURCE_MEMORY_MB = TEZ_AM_PREFIX + "resource.memory.mb"; public static final int TEZ_AM_RESOURCE_MEMORY_MB_DEFAULT = 1024; /** Int value. The number of virtual cores to be used by the app master */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_AM_RESOURCE_CPU_VCORES = TEZ_AM_PREFIX + "resource.cpu.vcores"; public static final int TEZ_AM_RESOURCE_CPU_VCORES_DEFAULT = 1; /** Boolean value. Instructs AM to delete Dag directory upon completion */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_AM_DAG_CLEANUP_ON_COMPLETION = TEZ_AM_PREFIX + "dag.cleanup.on.completion"; public static final boolean TEZ_AM_DAG_CLEANUP_ON_COMPLETION_DEFAULT = false; @@ -894,7 +891,7 @@ public TezConfiguration(boolean loadDefaults) { * when Reduce 3 completes all the shuffle data of Map1 will be deleted and so on for other vertex. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_AM_VERTEX_CLEANUP_HEIGHT = TEZ_AM_PREFIX + "vertex.cleanup.height"; public static final int TEZ_AM_VERTEX_CLEANUP_HEIGHT_DEFAULT = 0; @@ -903,9 +900,9 @@ public TezConfiguration(boolean loadDefaults) { * Boolean value. Instructs AM to delete intermediate attempt data for failed task attempts. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_AM_TASK_ATTEMPT_CLEANUP_ON_FAILURE = TEZ_AM_PREFIX - + "task.attempt.cleanup.on.failure"; + + "task.attempt.cleanup.on.failure"; public static final boolean TEZ_AM_TASK_ATTEMPT_CLEANUP_ON_FAILURE_DEFAULT = false; /** @@ -913,7 +910,7 @@ public TezConfiguration(boolean loadDefaults) { * Vertex directories and failed task attempts directories on nodes. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_AM_DAG_CLEANUP_THREAD_COUNT_LIMIT = TEZ_AM_PREFIX + "dag.deletion.thread-count-limit"; @@ -923,9 +920,9 @@ public TezConfiguration(boolean loadDefaults) { * Int value. Upper limit on the number of threads used by app context (vertex management and input init events). */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_AM_DAG_APPCONTEXT_THREAD_COUNT_LIMIT = - TEZ_AM_PREFIX + "dag.appcontext.thread-count-limit"; + TEZ_AM_PREFIX + "dag.appcontext.thread-count-limit"; public static final int TEZ_AM_DAG_APPCONTEXT_THREAD_COUNT_LIMIT_DEFAULT = 10; @@ -933,7 +930,7 @@ public TezConfiguration(boolean loadDefaults) { * all vertices. Setting it to the same value for all tasks is helpful for container reuse and * thus good for performance typically. */ @ConfigurationScope(Scope.DAG) // TODO vertex level - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_TASK_RESOURCE_MEMORY_MB = TEZ_TASK_PREFIX + "resource.memory.mb"; public static final int TEZ_TASK_RESOURCE_MEMORY_MB_DEFAULT = 1024; @@ -942,7 +939,7 @@ public TezConfiguration(boolean loadDefaults) { * Int value. The number of virtual cores to be used by tasks. */ @ConfigurationScope(Scope.DAG) // TODO vertex level - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_TASK_RESOURCE_CPU_VCORES = TEZ_TASK_PREFIX + "resource.cpu.vcores"; public static final int TEZ_TASK_RESOURCE_CPU_VCORES_DEFAULT = 1; @@ -953,7 +950,7 @@ public TezConfiguration(boolean loadDefaults) { * help in scaling up. Expert level setting. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX = TEZ_AM_PREFIX + "am-rm.heartbeat.interval-ms.max"; public static final int TEZ_AM_RM_HEARTBEAT_INTERVAL_MS_MAX_DEFAULT = 1000; @@ -964,7 +961,7 @@ public TezConfiguration(boolean loadDefaults) { * number of concurrent tasks. Expert level setting. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_TASK_GET_TASK_SLEEP_INTERVAL_MS_MAX = TEZ_TASK_PREFIX + "get-task.sleep.interval-ms.max"; public static final int TEZ_TASK_GET_TASK_SLEEP_INTERVAL_MS_MAX_DEFAULT = 200; @@ -975,7 +972,7 @@ public TezConfiguration(boolean loadDefaults) { * Expert level setting. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_TASK_AM_HEARTBEAT_INTERVAL_MS = TEZ_TASK_PREFIX + "am.heartbeat.interval-ms.max"; public static final int TEZ_TASK_AM_HEARTBEAT_INTERVAL_MS_DEFAULT = 100; @@ -986,7 +983,7 @@ public TezConfiguration(boolean loadDefaults) { * counters. Improves AM scalability. Expert level setting. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_TASK_AM_HEARTBEAT_COUNTER_INTERVAL_MS = TEZ_TASK_PREFIX + "am.heartbeat.counter.interval-ms.max"; public static final int TEZ_TASK_AM_HEARTBEAT_COUNTER_INTERVAL_MS_DEFAULT = @@ -997,7 +994,7 @@ public TezConfiguration(boolean loadDefaults) { * Expert level setting. Expert level setting. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_TASK_MAX_EVENTS_PER_HEARTBEAT = TEZ_TASK_PREFIX + "max-events-per-heartbeat"; public static final int TEZ_TASK_MAX_EVENTS_PER_HEARTBEAT_DEFAULT = 500; @@ -1008,7 +1005,7 @@ public TezConfiguration(boolean loadDefaults) { * Expert level setting. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_TASK_MAX_EVENT_BACKLOG = TEZ_TASK_PREFIX + "max-event-backlog"; public static final int TEZ_TASK_MAX_EVENT_BACKLOG_DEFAULT = 10000; @@ -1019,7 +1016,7 @@ public TezConfiguration(boolean loadDefaults) { * Expert level setting. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_TASK_INITIALIZE_PROCESSOR_FIRST = TEZ_TASK_PREFIX + "initialize-processor-first"; public static final boolean TEZ_TASK_INITIALIZE_PROCESSOR_FIRST_DEFAULT = false; @@ -1030,7 +1027,7 @@ public TezConfiguration(boolean loadDefaults) { * Expert level setting. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_TASK_INITIALIZE_PROCESSOR_IO_SERIALLY = TEZ_TASK_PREFIX + "initialize-processor-io-serially"; public static final boolean TEZ_TASK_INITIALIZE_PROCESSOR_IO_SERIALLY_DEFAULT = false; @@ -1046,7 +1043,7 @@ public TezConfiguration(boolean loadDefaults) { @ConfigurationScope(Scope.VERTEX) @ConfigurationProperty public static final String TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS = TEZ_TASK_PREFIX + - "progress.stuck.interval-ms"; + "progress.stuck.interval-ms"; public static final long TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS_DEFAULT = -1; /** @@ -1057,7 +1054,7 @@ public TezConfiguration(boolean loadDefaults) { @Unstable @Private @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_TASK_GENERATE_COUNTERS_PER_IO = TEZ_TASK_PREFIX + "generate.counters.per.io"; @Private @@ -1069,7 +1066,7 @@ public TezConfiguration(boolean loadDefaults) { * Expert level setting. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TASK_HEARTBEAT_TIMEOUT_MS = TEZ_TASK_PREFIX + "timeout-ms"; public static final int TASK_HEARTBEAT_TIMEOUT_MS_DEFAULT = 5 * 60 * 1000; @@ -1079,7 +1076,7 @@ public TezConfiguration(boolean loadDefaults) { * Expert level setting. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TASK_HEARTBEAT_TIMEOUT_CHECK_MS = TEZ_TASK_PREFIX + "heartbeat.timeout.check-ms"; public static final int TASK_HEARTBEAT_TIMEOUT_CHECK_MS_DEFAULT = 30 * 1000; @@ -1091,7 +1088,7 @@ public TezConfiguration(boolean loadDefaults) { @Private @Unstable @ConfigurationScope(Scope.VERTEX) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_TASK_SCALE_MEMORY_ENABLED = TEZ_TASK_PREFIX + "scale.memory.enabled"; @Private @@ -1117,7 +1114,7 @@ public TezConfiguration(boolean loadDefaults) { @Private @Unstable @ConfigurationScope(Scope.VERTEX) - @ConfigurationProperty(type="double") + @ConfigurationProperty(type = "double") public static final String TEZ_TASK_SCALE_MEMORY_RESERVE_FRACTION = TEZ_TASK_PREFIX + "scale.memory.reserve-fraction"; @Private @@ -1130,7 +1127,7 @@ public TezConfiguration(boolean loadDefaults) { @Private @Unstable @ConfigurationScope(Scope.VERTEX) - @ConfigurationProperty(type="float") + @ConfigurationProperty(type = "float") public static final String TEZ_TASK_SCALE_MEMORY_ADDITIONAL_RESERVATION_FRACTION_PER_IO = TEZ_TASK_PREFIX + "scale.memory.additional-reservation.fraction.per-io"; @@ -1140,7 +1137,7 @@ public TezConfiguration(boolean loadDefaults) { * Max cumulative total reservation for additional IOs. */ @ConfigurationScope(Scope.VERTEX) - @ConfigurationProperty(type="float") + @ConfigurationProperty(type = "float") public static final String TEZ_TASK_SCALE_MEMORY_ADDITIONAL_RESERVATION_FRACTION_MAX = TEZ_TASK_PREFIX + "scale.memory.additional-reservation.fraction.max"; /* @@ -1195,13 +1192,12 @@ public TezConfiguration(boolean loadDefaults) { public static final String TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS = TEZ_TASK_PREFIX + "resource.calculator.process-tree.class"; - /** * Boolean value. Configuration to specify whether container should be reused across tasks. * This improves performance by not incurring recurring launch overheads. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_AM_CONTAINER_REUSE_ENABLED = TEZ_AM_PREFIX + "container.reuse.enabled"; public static final boolean TEZ_AM_CONTAINER_REUSE_ENABLED_DEFAULT = true; @@ -1211,7 +1207,7 @@ public TezConfiguration(boolean loadDefaults) { * enabled. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED = TEZ_AM_PREFIX + "container.reuse.rack-fallback.enabled"; public static final boolean @@ -1223,7 +1219,7 @@ public TezConfiguration(boolean loadDefaults) { * volume being read from the primary data sources. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED = TEZ_AM_PREFIX + "container.reuse.non-local-fallback.enabled"; public static final boolean @@ -1237,7 +1233,7 @@ public TezConfiguration(boolean loadDefaults) { * be released. Active only if container reuse is enabled. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_AM_CONTAINER_REUSE_NEW_CONTAINERS_ENABLED = TEZ_AM_PREFIX + "container.reuse.new-containers.enabled"; public static final boolean @@ -1248,23 +1244,23 @@ public TezConfiguration(boolean loadDefaults) { * of locality. NODE -> RACK -> NON_LOCAL. Delay scheduling parameter. Expert level setting. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="long") + @ConfigurationProperty(type = "long") public static final String TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS = TEZ_AM_PREFIX + "container.reuse.locality.delay-allocation-millis"; public static final long - TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS_DEFAULT = 250l; + TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS_DEFAULT = 250l; /** * Int value. The minimum amount of time to hold on to a container that is idle. Only active when * reuse is enabled. Set to -1 to never release idle containers (not recommended). */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS = - TEZ_AM_PREFIX + "container.idle.release-timeout-min.millis"; + TEZ_AM_PREFIX + "container.idle.release-timeout-min.millis"; public static final long - TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS_DEFAULT = 5000l; + TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS_DEFAULT = 5000l; /** * Int value. The maximum amount of time to hold on to a container if no task can be @@ -1277,11 +1273,11 @@ public TezConfiguration(boolean loadDefaults) { * creates a graceful reduction in the amount of idle resources held */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="long") + @ConfigurationProperty(type = "long") public static final String TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS = TEZ_AM_PREFIX + "container.idle.release-timeout-max.millis"; public static final long - TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS_DEFAULT = 10000l; + TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS_DEFAULT = 10000l; /** * Int value. The minimum number of containers that will be held in session mode. Not active in @@ -1289,7 +1285,7 @@ public TezConfiguration(boolean loadDefaults) { * of containers to provide fast response times for the next DAG. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_AM_SESSION_MIN_HELD_CONTAINERS = TEZ_AM_PREFIX + "session.min.held-containers"; public static final int TEZ_AM_SESSION_MIN_HELD_CONTAINERS_DEFAULT = 0; @@ -1313,7 +1309,7 @@ public TezConfiguration(boolean loadDefaults) { * setting. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_AM_PREEMPTION_PERCENTAGE = TEZ_AM_PREFIX + "preemption.percentage"; public static final int TEZ_AM_PREEMPTION_PERCENTAGE_DEFAULT = 10; @@ -1324,7 +1320,7 @@ public TezConfiguration(boolean loadDefaults) { */ @ConfigurationScope(Scope.VERTEX) public static final String TEZ_VERTEX_FAILURES_MAXPERCENT = - "tez.vertex.failures.maxpercent"; + "tez.vertex.failures.maxpercent"; public static final float TEZ_VERTEX_FAILURES_MAXPERCENT_DEFAULT = 0.0f; /** * Int value. The number of RM heartbeats to wait after preempting running tasks before preempting @@ -1332,7 +1328,7 @@ public TezConfiguration(boolean loadDefaults) { * RM can act on the released resources and assign new ones to us. Expert level setting. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS = TEZ_AM_PREFIX + "preemption.heartbeats-between-preemptions"; public static final int TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS_DEFAULT = 3; @@ -1344,10 +1340,10 @@ public TezConfiguration(boolean loadDefaults) { * a deadline on such wait to prevent indefinite job hangs. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_AM_PREEMPTION_MAX_WAIT_TIME_MS = TEZ_AM_PREFIX + "preemption.max.wait-time-ms"; - public static final int TEZ_AM_PREEMPTION_MAX_WAIT_TIME_MS_DEFAULT = 60*1000; // 60s + public static final int TEZ_AM_PREEMPTION_MAX_WAIT_TIME_MS_DEFAULT = 60 * 1000; // 60s /** * String value to a file path. @@ -1410,7 +1406,7 @@ public TezConfiguration(boolean loadDefaults) { */ @Unstable @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_IGNORE_LIB_URIS = TEZ_PREFIX + "ignore.lib.uris"; /** @@ -1420,7 +1416,7 @@ public TezConfiguration(boolean loadDefaults) { * tez-deployment which contains the hadoop libraries. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_USE_CLUSTER_HADOOP_LIBS = TEZ_PREFIX + "use.cluster.hadoop-libs"; public static final boolean TEZ_USE_CLUSTER_HADOOP_LIBS_DEFAULT = false; @@ -1430,7 +1426,7 @@ public TezConfiguration(boolean loadDefaults) { * classpath. */ @ConfigurationScope(Scope.CLIENT) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_USER_CLASSPATH_FIRST = TEZ_PREFIX + "user.classpath.first"; public static final boolean TEZ_USER_CLASSPATH_FIRST_DEFAULT = true; @@ -1455,7 +1451,7 @@ public TezConfiguration(boolean loadDefaults) { @Private @Unstable @ConfigurationScope(Scope.CLIENT) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_CLASSPATH_ADD_HADOOP_CONF = TEZ_PREFIX + "classpath.add-hadoop-conf"; public static final boolean TEZ_CLASSPATH_ADD_HADOOP_CONF_DEFAULT = false; @@ -1475,7 +1471,7 @@ public TezConfiguration(boolean loadDefaults) { * applicable. Otherwise increase the timeout (set to -1 for infinity. Not recommended) */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_SESSION_CLIENT_TIMEOUT_SECS = TEZ_SESSION_PREFIX + "client.timeout.secs"; public static final int TEZ_SESSION_CLIENT_TIMEOUT_SECS_DEFAULT = @@ -1487,7 +1483,7 @@ public TezConfiguration(boolean loadDefaults) { * allow the AM to hang around forever in idle mode. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS = TEZ_SESSION_PREFIX + "am.dag.submit.timeout.secs"; public static final int TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS_DEFAULT = @@ -1519,7 +1515,7 @@ public TezConfiguration(boolean loadDefaults) { * Boolean value. Generate debug artifacts like DAG plan in text format. */ @ConfigurationScope(Scope.DAG) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_GENERATE_DEBUG_ARTIFACTS = TEZ_PREFIX + "generate.debug.artifacts"; public static final boolean TEZ_GENERATE_DEBUG_ARTIFACTS_DEFAULT = false; @@ -1636,7 +1632,7 @@ public TezConfiguration(boolean loadDefaults) { @Private @Unstable @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_HISTORY_LOGGING_TIMELINE_NUM_DAGS_PER_GROUP = TEZ_PREFIX + "history.logging.timeline.num-dags-per-group"; public static final int TEZ_HISTORY_LOGGING_TIMELINE_NUM_DAGS_PER_GROUP_DEFAULT = 1; @@ -1656,7 +1652,7 @@ public TezConfiguration(boolean loadDefaults) { * logging gets disabled. The job continues to run after this. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_SIMPLE_HISTORY_LOGGING_MAX_ERRORS = TEZ_PREFIX + "simple.history.max.errors"; public static final int TEZ_SIMPLE_HISTORY_LOGGING_MAX_ERRORS_DEFAULT = 10; @@ -1677,7 +1673,7 @@ public TezConfiguration(boolean loadDefaults) { * content and metadata. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="long") + @ConfigurationProperty(type = "long") public static final String TEZ_HISTORY_LOGGING_PROTO_SYNC_WINDOWN_SECS = TEZ_PREFIX + "history.logging.proto-sync-window-secs"; public static final long TEZ_HISTORY_LOGGING_PROTO_SYNC_WINDOWN_SECS_DEFAULT = 60L; @@ -1686,19 +1682,18 @@ public TezConfiguration(boolean loadDefaults) { * Int value. Maximum queue size for proto history event logger. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_HISTORY_LOGGING_PROTO_QUEUE_SIZE = TEZ_PREFIX + "history.logging.queue.size"; public static final int TEZ_HISTORY_LOGGING_PROTO_QUEUE_SIZE_DEFAULT = 100000; - /** * Boolean value. Set this to true, if the underlying file system does not support flush (Ex: s3). * The dag submitted, initialized and started events are written into a file and closed. The rest * of the events are written into another file. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_HISTORY_LOGGING_PROTO_SPLIT_DAG_START = TEZ_PREFIX + "history.logging.split-dag-start"; public static final boolean TEZ_HISTORY_LOGGING_PROTO_SPLIT_DAG_START_DEFAULT = false; @@ -1709,7 +1704,7 @@ public TezConfiguration(boolean loadDefaults) { * content and metadata. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_HISTORY_LOGGING_PROTO_DOAS = TEZ_PREFIX + "history.logging.proto-doas"; public static final boolean TEZ_HISTORY_LOGGING_PROTO_DOAS_DEFAULT = false; @@ -1719,7 +1714,7 @@ public TezConfiguration(boolean loadDefaults) { * Expert level setting. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="long") + @ConfigurationProperty(type = "long") public static final String YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS = TEZ_PREFIX + "yarn.ats.event.flush.timeout.millis"; public static final long YARN_ATS_EVENT_FLUSH_TIMEOUT_MILLIS_DEFAULT = @@ -1730,7 +1725,7 @@ public TezConfiguration(boolean loadDefaults) { * Expert level setting. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String YARN_ATS_MAX_EVENTS_PER_BATCH = TEZ_PREFIX + "yarn.ats.max.events.per.batch"; public static final int YARN_ATS_MAX_EVENTS_PER_BATCH_DEFAULT = 5; @@ -1747,19 +1742,18 @@ public TezConfiguration(boolean loadDefaults) { TEZ_PREFIX + "am.ui.history.url.scheme.check.enabled"; public static final boolean TEZ_AM_UI_HISTORY_URL_SCHEME_CHECK_ENABLED_DEFAULT = true; - /** * Int value. Time, in milliseconds, to wait for an event before sending a batch to ATS. * Expert level setting. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String YARN_ATS_MAX_POLLING_TIME_PER_EVENT = TEZ_PREFIX + "yarn.ats.max.polling.time.per.event.millis"; public static final int YARN_ATS_MAX_POLLING_TIME_PER_EVENT_DEFAULT = 10; @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String YARN_ATS_ACL_DOMAINS_AUTO_CREATE = TEZ_PREFIX + "yarn.ats.acl.domains.auto-create"; public static final boolean YARN_ATS_ACL_DOMAINS_AUTO_CREATE_DEFAULT = true; @@ -1776,7 +1770,7 @@ public TezConfiguration(boolean loadDefaults) { * incomplete DAGs from the previous instance of the app master. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String DAG_RECOVERY_ENABLED = TEZ_PREFIX + "dag.recovery.enabled"; public static final boolean DAG_RECOVERY_ENABLED_DEFAULT = true; @@ -1786,7 +1780,7 @@ public TezConfiguration(boolean loadDefaults) { * Expert level setting. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String DAG_RECOVERY_FILE_IO_BUFFER_SIZE = TEZ_PREFIX + "dag.recovery.io.buffer.size"; public static final int DAG_RECOVERY_FILE_IO_BUFFER_SIZE_DEFAULT = 8192; @@ -1795,7 +1789,7 @@ public TezConfiguration(boolean loadDefaults) { * Int value. Number of recovery events to buffer before flushing them to the recovery log. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String DAG_RECOVERY_MAX_UNFLUSHED_EVENTS = TEZ_PREFIX + "dag.recovery.max.unflushed.events"; public static final int DAG_RECOVERY_MAX_UNFLUSHED_EVENTS_DEFAULT = 100; @@ -1804,7 +1798,7 @@ public TezConfiguration(boolean loadDefaults) { * Int value. Interval, in seconds, between flushing recovery data to the recovery log. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String DAG_RECOVERY_FLUSH_INTERVAL_SECS = TEZ_PREFIX + "dag.recovery.flush.interval.secs"; public static final int DAG_RECOVERY_FLUSH_INTERVAL_SECS_DEFAULT = 30; @@ -1814,18 +1808,18 @@ public TezConfiguration(boolean loadDefaults) { * the app master. Primarily used for debugging. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_LOCAL_MODE = - TEZ_PREFIX + "local.mode"; + TEZ_PREFIX + "local.mode"; public static final boolean TEZ_LOCAL_MODE_DEFAULT = false; /** - * Boolean value. Enable local mode execution in Tez without using network for communicating with - * DAGAppMaster. This option only makes sense when {@link #TEZ_LOCAL_MODE} is true. When - * TEZ_LOCAL_MODE_WITHOUT_NETWORK is turned on, LocalClient will call DAGAppMaster's methods - * directly. - */ + * Boolean value. Enable local mode execution in Tez without using network for communicating with + * DAGAppMaster. This option only makes sense when {@link #TEZ_LOCAL_MODE} is true. When + * TEZ_LOCAL_MODE_WITHOUT_NETWORK is turned on, LocalClient will call DAGAppMaster's methods + * directly. + */ @ConfigurationScope(Scope.AM) @ConfigurationProperty(type = "boolean") public static final String TEZ_LOCAL_MODE_WITHOUT_NETWORK = @@ -1847,9 +1841,9 @@ public TezConfiguration(boolean loadDefaults) { */ @Private @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_AM_INLINE_TASK_EXECUTION_ENABLED = - TEZ_AM_PREFIX + "inline.task.execution.enabled"; + TEZ_AM_PREFIX + "inline.task.execution.enabled"; /** * Tez AM Inline Mode flag. Not valid till Tez-684 get checked-in @@ -1862,9 +1856,9 @@ public TezConfiguration(boolean loadDefaults) { * The maximium number of tasks running in parallel within the app master process. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS = - TEZ_AM_PREFIX + "inline.task.execution.max-tasks"; + TEZ_AM_PREFIX + "inline.task.execution.max-tasks"; public static final int TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS_DEFAULT = 1; @@ -1888,7 +1882,7 @@ public TezConfiguration(boolean loadDefaults) { * Boolean value. Configuration to enable/disable ACL checks. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_AM_ACLS_ENABLED = TEZ_AM_PREFIX + "acls.enabled"; public static final boolean TEZ_AM_ACLS_ENABLED_DEFAULT = true; @@ -1920,7 +1914,7 @@ public TezConfiguration(boolean loadDefaults) { */ @Private @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_AM_DISABLE_CLIENT_VERSION_CHECK = TEZ_AM_PREFIX + "disable.client-version-check"; public static final boolean TEZ_AM_DISABLE_CLIENT_VERSION_CHECK_DEFAULT = false; @@ -1931,7 +1925,7 @@ public TezConfiguration(boolean loadDefaults) { */ @Private @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS = TEZ_PREFIX + "allow.disabled.timeline-domains"; public static final boolean TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS_DEFAULT = false; @@ -1975,7 +1969,7 @@ public TezConfiguration(boolean loadDefaults) { * updates for running application. */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_AM_WEBSERVICE_ENABLE = TEZ_AM_PREFIX + "tez-ui.webservice.enable"; public static final boolean TEZ_AM_WEBSERVICE_ENABLE_DEFAULT = true; @@ -1986,7 +1980,7 @@ public TezConfiguration(boolean loadDefaults) { * For example 50000-50050,50100-50200 */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_AM_WEBSERVICE_PORT_RANGE = TEZ_AM_PREFIX + "tez-ui.webservice.port-range"; @@ -2015,7 +2009,7 @@ static Set getPropertySet() { * Status Poll interval in Milliseconds used when getting DAG status with timeout. */ @ConfigurationScope(Scope.DAG) - @ConfigurationProperty(type="long") + @ConfigurationProperty(type = "long") public static final String TEZ_DAG_STATUS_POLLINTERVAL_MS = TEZ_PREFIX + "dag.status.pollinterval-ms"; public static final long TEZ_DAG_STATUS_POLLINTERVAL_MS_DEFAULT = 500; @@ -2026,7 +2020,7 @@ static Set getPropertySet() { */ @Private @ConfigurationScope(Scope.TEST) - @ConfigurationProperty(type="long") + @ConfigurationProperty(type = "long") public static final String TEZ_TEST_MINI_CLUSTER_APP_WAIT_ON_SHUTDOWN_SECS = TEZ_PREFIX + "test.minicluster.app.wait.on.shutdown.secs"; public static final long TEZ_TEST_MINI_CLUSTER_APP_WAIT_ON_SHUTDOWN_SECS_DEFAULT = 30; @@ -2037,7 +2031,7 @@ static Set getPropertySet() { */ @Private @ConfigurationScope(Scope.CLIENT) - @ConfigurationProperty(type="long") + @ConfigurationProperty(type = "long") public static final String TEZ_CLIENT_DAG_STATUS_CACHE_TIMEOUT_SECS = TEZ_PREFIX + "client.dag.status.cache.timeout-secs"; // Default timeout is 60 seconds. @@ -2050,11 +2044,11 @@ static Set getPropertySet() { */ @Private @ConfigurationScope(Scope.CLIENT) - @ConfigurationProperty(type="long") + @ConfigurationProperty(type = "long") public static final String TEZ_CLIENT_DIAGNOSTICS_WAIT_TIMEOUT_MS = TEZ_PREFIX + "client.diagnostics.wait.timeout-ms"; @Private - public static final long TEZ_CLIENT_DIAGNOSTICS_WAIT_TIMEOUT_MS_DEFAULT = 3*1000; + public static final long TEZ_CLIENT_DIAGNOSTICS_WAIT_TIMEOUT_MS_DEFAULT = 3 * 1000; /** * String value. @@ -2074,7 +2068,7 @@ static Set getPropertySet() { */ @Private @ConfigurationScope(Scope.CLIENT) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_CLIENT_JAVA_OPTS_CHECKER_ENABLED = TEZ_PREFIX + "java.opts.checker.enabled"; public static final boolean TEZ_CLIENT_JAVA_OPTS_CHECKER_ENABLED_DEFAULT = true; @@ -2085,7 +2079,7 @@ static Set getPropertySet() { * Expert level setting. */ @ConfigurationScope(Scope.CLIENT) - @ConfigurationProperty(type="long") + @ConfigurationProperty(type = "long") public static final String TEZ_CLIENT_HARD_KILL_TIMEOUT_MS = TEZ_PREFIX + "client.timeout-ms"; public static final long TEZ_CLIENT_HARD_KILL_TIMEOUT_MS_DEFAULT = 30 * 1000L; @@ -2095,7 +2089,7 @@ static Set getPropertySet() { * Expert level setting. */ @ConfigurationScope(Scope.CLIENT) - @ConfigurationProperty(type="boolean") + @ConfigurationProperty(type = "boolean") public static final String TEZ_CLIENT_ASYNCHRONOUS_STOP = TEZ_PREFIX + "client.asynchronous-stop"; public static final boolean TEZ_CLIENT_ASYNCHRONOUS_STOP_DEFAULT = true; @@ -2105,7 +2099,7 @@ static Set getPropertySet() { */ @Private @ConfigurationScope(Scope.CLIENT) - @ConfigurationProperty(type="int") + @ConfigurationProperty(type = "int") public static final String TEZ_IPC_PAYLOAD_RESERVED_BYTES = TEZ_PREFIX + "ipc.payload.reserved.bytes"; public static final int TEZ_IPC_PAYLOAD_RESERVED_BYTES_DEFAULT = 5 * 1024 * 1024; @@ -2140,16 +2134,16 @@ static Set getPropertySet() { TEZ_PREFIX + "am.ats.v15.override.summary-types"; public static final boolean TEZ_AM_ATS_V15_OVERRIDE_SUMMARY_TYPES_DEFAULT = true; - /** - * Integer value in milliseconds. Default value is 5000 milliseconds. - * The time for which the AM waits after the final DAG completes or when shutdown is invoked - * before completing shutdown. This allows a client to retrieve any required info directly from - * the AM on completion of a DAG. - */ - @Private - @ConfigurationScope(Scope.AM) - public static final String TEZ_AM_SLEEP_TIME_BEFORE_EXIT_MILLIS = - TEZ_AM_PREFIX + "sleep.time.before.exit.millis"; + /** + * Integer value in milliseconds. Default value is 5000 milliseconds. + * The time for which the AM waits after the final DAG completes or when shutdown is invoked + * before completing shutdown. This allows a client to retrieve any required info directly from + * the AM on completion of a DAG. + */ + @Private + @ConfigurationScope(Scope.AM) + public static final String TEZ_AM_SLEEP_TIME_BEFORE_EXIT_MILLIS = + TEZ_AM_PREFIX + "sleep.time.before.exit.millis"; /** * String value. Determines what JVM properties will be logged for debugging purposes @@ -2157,13 +2151,13 @@ static Set getPropertySet() { */ @ConfigurationScope(Scope.AM) @ConfigurationProperty - public static final String TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG = + public static final String TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG = TEZ_PREFIX + "tez.jvm.system-properties-to-log"; public static final List TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG_DEFAULT = Collections.unmodifiableList(Arrays.asList( - "os.name","os.version","java.home","java.runtime.version", - "java.vendor","java.version","java.vm.name","java.class.path", - "java.io.tmpdir","user.dir","user.name")); + "os.name", "os.version", "java.home", "java.runtime.version", + "java.vendor", "java.version", "java.vm.name", "java.class.path", + "java.io.tmpdir", "user.dir", "user.name")); /** * Int value. Time interval (in seconds). If the Tez AM does not receive a heartbeat from the @@ -2177,12 +2171,11 @@ static Set getPropertySet() { * {@link #TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS} */ @ConfigurationScope(Scope.AM) - @ConfigurationProperty(type="integer") + @ConfigurationProperty(type = "integer") public static final String TEZ_AM_CLIENT_HEARTBEAT_TIMEOUT_SECS = TEZ_PREFIX + "am.client.heartbeat.timeout.secs"; public static final int TEZ_AM_CLIENT_HEARTBEAT_TIMEOUT_SECS_DEFAULT = -1; - @Private @ConfigurationScope(Scope.AM) public static final String TEZ_AM_CLIENT_HEARTBEAT_POLL_INTERVAL_MILLIS = @@ -2227,5 +2220,4 @@ static Set getPropertySet() { @ConfigurationScope(Scope.VERTEX) @ConfigurationProperty public static final String TEZ_MRREADER_CONFIG_UPDATE_PROPERTIES = "tez.mrreader.config.update.properties"; - } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfigurationConstants.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfigurationConstants.java index 33abc77de4..7c3ad1ecc5 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfigurationConstants.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfigurationConstants.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -51,5 +51,4 @@ public class TezConfigurationConstants { @ConfigurationProperty public static final String TEZ_SUBMIT_HOST_ADDRESS = TezConfiguration.TEZ_PREFIX + "submit.host.address"; - } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java index cf5ab11ec7..f83fb4c262 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,27 +26,26 @@ @Private public class TezConstants { - public static final String TEZ_APPLICATION_MASTER_CLASS = "org.apache.tez.dag.app.DAGAppMaster"; - + /** * Command-line argument to be set when running the Tez AM in session mode. */ public static final String TEZ_SESSION_MODE_CLI_OPTION = "session"; public static final String TEZ_TAR_LR_NAME = "tezlib"; - + /* * Tez AM Service Authorization * These are the same as MR which allows Tez to run in secure * mode without configuring service ACLs */ - public static final String - TEZ_AM_SECURITY_SERVICE_AUTHORIZATION_TASK_UMBILICAL = + public static final String + TEZ_AM_SECURITY_SERVICE_AUTHORIZATION_TASK_UMBILICAL = "security.job.task.protocol.acl"; - public static final String - TEZ_AM_SECURITY_SERVICE_AUTHORIZATION_CLIENT = + public static final String + TEZ_AM_SECURITY_SERVICE_AUTHORIZATION_CLIENT = "security.job.client.protocol.acl"; public static final String TEZ_PB_BINARY_CONF_NAME = "tez-conf.pb"; @@ -66,8 +65,8 @@ public class TezConstants { public static final String TEZ_CONTAINER_LOG_PARAMS_SEPARATOR = ";"; public static final String TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME = - TezConfiguration.TEZ_SESSION_PREFIX + "local-resources.pb"; - + TezConfiguration.TEZ_SESSION_PREFIX + "local-resources.pb"; + public static final String TEZ_APPLICATION_TYPE = "TEZ"; /** * The service id for the NodeManager plugin used to share intermediate data @@ -81,9 +80,8 @@ public class TezConstants { public static final String DAG_RECOVERY_SUMMARY_FILE_SUFFIX = "summary"; public static final String DAG_RECOVERY_RECOVER_FILE_SUFFIX = ".recovery"; - // Configuration keys used internally and not set by the users - + // These are session specific DAG ACL's. Currently here because these can only be specified // via code in the API. /** @@ -128,5 +126,4 @@ public static String getTezUberServicePluginName() { public static final int TEZ_CONTAINER_SMALL_SLAB_BOUND_MB = 4096; public static final double TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_SMALL_SLAB = 0.7; public static final double TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_LARGE_SLAB = 0.8; - } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezException.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezException.java index 29c70d737c..1ea8b67f3f 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezException.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezException.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.api; @@ -26,8 +26,11 @@ @Public public class TezException extends Exception { private static final long serialVersionUID = 6337442733802964447L; - public TezException(Throwable cause) { super(cause); } - public TezException(String message) { super(message); } + + public TezException(Throwable cause) {super(cause);} + + public TezException(String message) {super(message);} + public TezException(String message, Throwable cause) { super(message, cause); } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezReflectionException.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezReflectionException.java index 4d8d1e0763..eea5ee5766 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezReflectionException.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezReflectionException.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezUncheckedException.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezUncheckedException.java index af798b3484..2cd0ac4210 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezUncheckedException.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezUncheckedException.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.api; @@ -27,9 +27,11 @@ public class TezUncheckedException extends RuntimeException { private static final long serialVersionUID = -4956339297375386184L; - - public TezUncheckedException(Throwable cause) { super(cause); } - public TezUncheckedException(String message) { super(message); } + + public TezUncheckedException(Throwable cause) {super(cause);} + + public TezUncheckedException(String message) {super(message);} + public TezUncheckedException(String message, Throwable cause) { super(message, cause); } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/UserPayload.java b/tez-api/src/main/java/org/apache/tez/dag/api/UserPayload.java index 087b17ab9d..80058dfa35 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/UserPayload.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/UserPayload.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,6 +19,7 @@ package org.apache.tez.dag.api; import com.google.common.annotations.VisibleForTesting; + import java.nio.ByteBuffer; import javax.annotation.Nullable; @@ -73,6 +74,7 @@ public ByteBuffer getRawPayload() { // ByteBuffer - deepCopyAsArray and TezEntityDescriptor need to be looked at. return payload == EMPTY_BYTE ? null : payload.duplicate(); } + /** * Return the payload as a read-only ByteBuffer. * @return read-only ByteBuffer. diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java b/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java index 8ce3fb67e6..32f3a45c40 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -60,9 +60,9 @@ public class Vertex { private Map vertexConf = new HashMap(); private VertexExecutionContext vertexExecutionContext; private final Map> additionalInputs - = new HashMap>(); + = new HashMap>(); private final Map> additionalOutputs - = new HashMap>(); + = new HashMap>(); private VertexManagerPluginDescriptor vertexManagerPlugin; private final List inputVertices = new ArrayList(); @@ -72,7 +72,7 @@ public class Vertex { private final Map groupInputs = Maps.newHashMap(); private final List dataSources = Lists.newLinkedList(); private final List dataSinks = Lists.newLinkedList(); - + private String taskLaunchCmdOpts = ""; @InterfaceAudience.Private @@ -86,17 +86,16 @@ public class Vertex { private Vertex(String vertexName, ProcessorDescriptor processorDescriptor, int parallelism) { this(vertexName, processorDescriptor, parallelism, null, true); } - private Vertex(String vertexName, ProcessorDescriptor processorDescriptor) { this(vertexName, processorDescriptor, -1); } - + private Vertex(String vertexName, - ProcessorDescriptor processorDescriptor, - int parallelism, - Resource taskResource, - boolean allowIncomplete) { + ProcessorDescriptor processorDescriptor, + int parallelism, + Resource taskResource, + boolean allowIncomplete) { this.vertexName = vertexName; this.processorDescriptor = processorDescriptor; this.parallelism = parallelism; @@ -104,7 +103,7 @@ private Vertex(String vertexName, if (parallelism < -1) { throw new IllegalArgumentException( "Parallelism should be -1 if determined by the AM" - + ", otherwise should be >= 0"); + + ", otherwise should be >= 0"); } if (!allowIncomplete && taskResource == null) { throw new IllegalArgumentException("Resource cannot be null"); @@ -183,7 +182,6 @@ public static Vertex create(String vertexName, ProcessorDescriptor processorDesc return new Vertex(vertexName, processorDescriptor, parallelism); } - /** * Get the vertex name * @return vertex name @@ -209,7 +207,7 @@ public ProcessorDescriptor getProcessorDescriptor() { public int getParallelism() { return parallelism; } - + /** * Set the number of tasks for this vertex * @param parallelism Parallelism for this vertex @@ -237,7 +235,7 @@ public Vertex setLocationHint(VertexLocationHint locationHint) { if (locations == null) { return this; } - Preconditions.checkArgument((locations.size() == parallelism), + Preconditions.checkArgument((locations.size() == parallelism), "Locations array length must match the parallelism set for the vertex"); this.locationHint = locationHint; return this; @@ -303,27 +301,27 @@ public Map getConf() { * @param cmdOpts * @return this Vertex */ - public Vertex setTaskLaunchCmdOpts(String cmdOpts){ - this.taskLaunchCmdOpts = cmdOpts; - return this; + public Vertex setTaskLaunchCmdOpts(String cmdOpts) { + this.taskLaunchCmdOpts = cmdOpts; + return this; } - + /** * Specifies an external data source for a Vertex. This is meant to be used * when a Vertex reads Input directly from an external source

- * + * * For vertices which read data generated by another vertex - use the * {@link DAG addEdge} method. - * + * * If a vertex needs to use data generated by another vertex in the DAG and * also from an external source, a combination of this API and the DAG.addEdge * API can be used.

- * + * * Note: If more than one RootInput exists on a vertex, which generates events * which need to be routed, or generates information to set parallelism, a * custom vertex manager should be setup to handle this. Not using a custom * vertex manager for such a scenario will lead to a runtime failure. - * + * * @param inputName * the name of the input. This will be used when accessing the input * in the {@link LogicalIOProcessor} @@ -347,14 +345,14 @@ public Vertex addDataSource(String inputName, DataSourceDescriptor dataSourceDes /** * Specifies an external data sink for a Vertex. This is meant to be used when * a Vertex writes Output directly to an external destination.

- * + * * If an output of the vertex is meant to be consumed by another Vertex in the * DAG - use the {@link DAG addEdge} method. - * + * * If a vertex needs generate data to an external source as well as for * another Vertex in the DAG, a combination of this API and the DAG.addEdge * API can be used. - * + * * @param outputName * the name of the output. This will be used when accessing the * output in the {@link LogicalIOProcessor} @@ -388,7 +386,7 @@ Vertex addAdditionalDataSink(RootInputLeafOutput getGroupInputs() { return groupInputs; } - + void addGroupInput(String groupName, GroupInfo groupInputInfo) { if (groupInputs.put(groupName, groupInputInfo) != null) { throw new IllegalStateException( - "Vertex: " + getName() + - " already has group input with name:" + groupName); + "Vertex: " + getName() + + " already has group input with name:" + groupName); } } @@ -633,7 +630,7 @@ void addOutputVertex(Vertex outputVertex, Edge edge) { outputVertices.add(outputVertex); outputEdges.add(edge); } - + /** * Get the input vertices for this vertex * @return List of input vertices @@ -662,7 +659,7 @@ void setTaskResource(Resource resource) { public List getDataSources() { return dataSources; } - + @Private public List getDataSinks() { return dataSinks; @@ -680,7 +677,7 @@ List getInputEdges() { List getOutputEdges() { return outputEdges; } - + List> getInputs() { return Lists.newArrayList(additionalInputs.values()); } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/VertexGroup.java b/tez-api/src/main/java/org/apache/tez/dag/api/VertexGroup.java index 8041b34a9c..f5e36f0f19 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/VertexGroup.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/VertexGroup.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -47,7 +47,7 @@ static class GroupInfo { Set outputs = new HashSet(); // destination vertex name to merged input map Map edgeMergedInputs = Maps.newHashMap(); - + GroupInfo(String groupName, Vertex... vertices) { this.groupName = groupName; members = Sets.newHashSetWithExpectedSize(vertices.length); @@ -55,19 +55,22 @@ static class GroupInfo { members.add(v); } } + String getGroupName() { return groupName; } + Set getMembers() { return members; } + Set getOutputs() { return outputs; } } - + GroupInfo groupInfo; - + /** * Create an object representing a group of vertices * @param groupName name of the group @@ -86,26 +89,26 @@ Set getOutputs() { public String getGroupName() { return groupInfo.groupName; } - + /** * Add an common data sink to the group of vertices. * Refer to {@link Vertex#addDataSink(String, DataSinkDescriptor)} * @return this object for further chained method calls */ public VertexGroup addDataSink(String outputName, DataSinkDescriptor dataSinkDescriptor) { - RootInputLeafOutput leafOutput = + RootInputLeafOutput leafOutput = new RootInputLeafOutput(outputName, - dataSinkDescriptor.getOutputDescriptor(), dataSinkDescriptor.getOutputCommitterDescriptor()); + dataSinkDescriptor.getOutputDescriptor(), dataSinkDescriptor.getOutputCommitterDescriptor()); this.groupInfo.outputs.add(outputName); - + // also add output to its members for (Vertex member : getMembers()) { member.addAdditionalDataSink(leafOutput); } - + return this; } - + @Override public String toString() { return "[ VertexGroup: " + groupInfo.getGroupName() + "]" + ", members=" + groupInfo.members; @@ -114,15 +117,15 @@ public String toString() { GroupInfo getGroupInfo() { return groupInfo; } - + Set getMembers() { return groupInfo.members; } - + void addOutputVertex(Vertex outputVertex, GroupInputEdge edge) { this.groupInfo.edgeMergedInputs.put(outputVertex.getName(), edge.getMergedInput()); } - + @Override public int hashCode() { final int prime = 31; diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java b/tez-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java index c020dd230b..5ec14dc1cc 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,11 +29,10 @@ * close to the specified locations. */ @Public -public class VertexLocationHint { +public class VertexLocationHint { private final List taskLocationHints; - private VertexLocationHint(List taskLocationHints) { if (taskLocationHints != null) { this.taskLocationHints = Collections.unmodifiableList(taskLocationHints); diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPlugin.java b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPlugin.java index b66a66ac55..93f29d167f 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPlugin.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPlugin.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.api; @@ -46,7 +46,7 @@ public abstract class VertexManagerPlugin { * Crete an instance of the VertexManagerPlugin. Classes extending this to * create a VertexManagerPlugin, must provide the same constructor so that Tez * can create an instance of the class at runtime. - * + * * @param context * vertex manager plugin context which can be used to access the * payload, vertex properties, etc @@ -91,7 +91,7 @@ public void onVertexStarted(List completions) throws Exce } onVertexStarted(completionsMap); } - + @Deprecated /** * This has been replaced by @@ -111,7 +111,7 @@ public void onSourceTaskCompleted(String srcVertexName, Integer taskId) throws E * @throws Exception */ public void onSourceTaskCompleted(TaskAttemptIdentifier attempt) throws Exception { - onSourceTaskCompleted(attempt.getTaskIdentifier().getVertexIdentifier().getName(), + onSourceTaskCompleted(attempt.getTaskIdentifier().getVertexIdentifier().getName(), attempt.getTaskIdentifier().getIdentifier()); } @@ -130,7 +130,7 @@ public void onSourceTaskCompleted(TaskAttemptIdentifier attempt) throws Exceptio * @throws Exception */ public abstract void onRootVertexInitialized(String inputName, - InputDescriptor inputDescriptor, List events) throws Exception; + InputDescriptor inputDescriptor, List events) throws Exception; /** * Return ahe {@link org.apache.tez.dag.api.VertexManagerPluginContext} for this specific instance of @@ -141,7 +141,7 @@ public abstract void onRootVertexInitialized(String inputName, public final VertexManagerPluginContext getContext() { return this.context; } - + /** * Receive notifications on vertex state changes. *

@@ -152,7 +152,7 @@ public final VertexManagerPluginContext getContext() { * change occurred. *


This method may be invoked concurrently with {@link #onVertexStarted(Map)} etc. and * multi-threading/concurrency implications must be considered. - * + * * @param stateUpdate * an event indicating the name of the vertex, and it's updated * state. Additional information may be available for specific @@ -160,5 +160,5 @@ public final VertexManagerPluginContext getContext() { * {@link org.apache.tez.dag.api.event.VertexStateUpdate} */ public void onVertexStateUpdated(VertexStateUpdate stateUpdate) throws Exception { - } - } + } +} diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java index 0862f49034..6a38457f4f 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginContext.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.api; @@ -43,43 +43,44 @@ @Unstable @Public public interface VertexManagerPluginContext { - + public class ScheduleTaskRequest { int taskIndex; TaskLocationHint locationHint; public static ScheduleTaskRequest create(int taskIndex, @Nullable TaskLocationHint locationHint) { - return new ScheduleTaskRequest(taskIndex, locationHint); + return new ScheduleTaskRequest(taskIndex, locationHint); } - + private ScheduleTaskRequest(int taskIndex, @Nullable TaskLocationHint locationHint) { Preconditions.checkState(taskIndex >= 0); this.taskIndex = taskIndex; this.locationHint = locationHint; } - + public int getTaskIndex() { return taskIndex; } - + public TaskLocationHint getTaskLocationHint() { return locationHint; - } + } } - + @Deprecated public class TaskWithLocationHint { Integer taskIndex; TaskLocationHint locationHint; + public TaskWithLocationHint(Integer taskIndex, @Nullable TaskLocationHint locationHint) { this.taskIndex = Objects.requireNonNull(taskIndex); this.locationHint = locationHint; } - + public Integer getTaskIndex() { return taskIndex; } - + public TaskLocationHint getTaskLocationHint() { return locationHint; } @@ -91,14 +92,14 @@ public TaskLocationHint getTaskLocationHint() { * @return Map of source vertex name and edge property */ public Map getInputVertexEdgeProperties(); - + /** * Get the edge properties on the output edges of this vertex. The output edge * is represented by the destination vertex name * @return Map of destination vertex name and edge property */ public Map getOutputVertexEdgeProperties(); - + /** * Get a {@link VertexStatistics} object to find out execution statistics * about the given {@link Vertex}. @@ -116,26 +117,26 @@ public TaskLocationHint getTaskLocationHint() { * @return Vertex name */ public String getVertexName(); - + /** * Get the payload set for the plugin * @return user payload */ public UserPayload getUserPayload(); - + /** * Get the number of tasks in the given vertex * @param vertexName * @return Total number of tasks in this vertex */ public int getVertexNumTasks(String vertexName); - + /** * Get the resource allocated to a task of this vertex * @return Resource */ Resource getVertexTaskResource(); - + /** * Get the total resource allocated to this vertex. If the DAG is running in * a busy cluster then it may have no resources available dedicated to it. The @@ -143,13 +144,13 @@ public TaskLocationHint getTaskLocationHint() { * @return Resource */ Resource getTotalAvailableResource(); - + /** * Get the number of nodes in the cluster * @return Number of nodes */ int getNumClusterNodes(); - + /** * Set the new parallelism (number of tasks) of this vertex, * Map of source (input) vertices and edge managers to change the event routing @@ -164,9 +165,9 @@ public TaskLocationHint getTaskLocationHint() { */ @Deprecated public void setVertexParallelism(int parallelism, - @Nullable VertexLocationHint locationHint, - @Nullable Map sourceEdgeManagers, - @Nullable Map rootInputSpecUpdate); + @Nullable VertexLocationHint locationHint, + @Nullable Map sourceEdgeManagers, + @Nullable Map rootInputSpecUpdate); /** * API to reconfigure a {@link Vertex} that is reading root inputs based on @@ -185,7 +186,7 @@ public void setVertexParallelism(int parallelism, * times until any of the tasks of the vertex have been scheduled (by invoking * {@link #scheduleTasks(List)}. If needed, the original source edge * properties may be obtained via {@link #getInputVertexEdgeProperties()} - * + * * @param parallelism * New number of tasks in the vertex * @param locationHint @@ -202,10 +203,10 @@ public void setVertexParallelism(int parallelism, * for details. */ public void reconfigureVertex(int parallelism, - @Nullable VertexLocationHint locationHint, - @Nullable Map sourceEdgeProperties, - @Nullable Map rootInputSpecUpdate); - + @Nullable VertexLocationHint locationHint, + @Nullable Map sourceEdgeProperties, + @Nullable Map rootInputSpecUpdate); + /** * API to reconfigure a {@link Vertex} by changing its task parallelism. Task * parallelism is often accompanied by changing the {@link EdgeProperty} of @@ -215,7 +216,7 @@ public void reconfigureVertex(int parallelism, * times until any of the tasks of the vertex have been scheduled (by invoking * {@link #scheduleTasks(List)}. If needed, the original source edge * properties may be obtained via {@link #getInputVertexEdgeProperties()} - * + * * @param parallelism * New number of tasks in the vertex * @param locationHint @@ -227,8 +228,8 @@ public void reconfigureVertex(int parallelism, * corresponding source vertex name. */ public void reconfigureVertex(int parallelism, - @Nullable VertexLocationHint locationHint, - @Nullable Map sourceEdgeProperties); + @Nullable VertexLocationHint locationHint, + @Nullable Map sourceEdgeProperties); /** * API to reconfigure a {@link Vertex} that is reading root inputs based on @@ -252,15 +253,15 @@ public void reconfigureVertex(int parallelism, * New number of tasks in the vertex */ public void reconfigureVertex(@Nullable Map rootInputSpecUpdate, - @Nullable VertexLocationHint locationHint, - int parallelism); + @Nullable VertexLocationHint locationHint, + int parallelism); /** * Allows a VertexManagerPlugin to assign Events for Root Inputs - * + * * For regular Event Routing changes - the EdgeManager should be configured * via the setVertexParallelism method - * + * * @param inputName * The input name associated with the event * @param events @@ -280,7 +281,7 @@ public void reconfigureVertex(@Nullable Map rootInputSp * @param taskId id of a task of managed vertex */ public void sendEventToProcessor(Collection events, int taskId); - + @Deprecated /** * Replaced by {@link #scheduleTasks(List)} @@ -288,7 +289,7 @@ public void reconfigureVertex(@Nullable Map rootInputSp * @param tasks Indices of the tasks to be started */ public void scheduleVertexTasks(List tasks); - + /** * Notify the vertex to schedule the given tasks * @param tasks Identifier and metadata for the tasks to schedule @@ -305,7 +306,7 @@ public void reconfigureVertex(@Nullable Map rootInputSp /** * Set the placement hint for tasks in this vertex - * + * * @param locationHint */ public void setVertexLocationHint(VertexLocationHint locationHint); @@ -314,7 +315,7 @@ public void reconfigureVertex(@Nullable Map rootInputSp * @return DAG Attempt number */ public int getDAGAttemptNumber(); - + /** * Register to get notifications on updates to the specified vertex. Notifications will be sent * via {@link VertexManagerPlugin#onVertexStateUpdated(org.apache.tez.dag.api.event.VertexStateUpdate)} @@ -325,7 +326,7 @@ public void reconfigureVertex(@Nullable Map rootInputSp * @param stateSet the set of states for which notifications are required. null implies all */ void registerForVertexStateUpdates(String vertexName, @Nullable Set stateSet); - + /** * Optional API. No need to call this when the vertex is not fully defined to * start with. E.g. vertex parallelism is not defined, or edges are not @@ -344,13 +345,13 @@ public void reconfigureVertex(@Nullable Map rootInputSp * reconfiguration. */ public void vertexReconfigurationPlanned(); - + /** * Optional API. This needs to be called only if {@link #vertexReconfigurationPlanned()} has been * invoked. This must be called after {@link #vertexReconfigurationPlanned()} is called. */ public void doneReconfiguringVertex(); - + /** * Optional API. This API can be invoked to declare that the * {@link VertexManagerPlugin} is done with its work. After this the system @@ -370,5 +371,4 @@ public void reconfigureVertex(@Nullable Map rootInputSp * or empty map if there is no input vertex group. */ Map> getInputVertexGroups(); - -} \ No newline at end of file +} diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginDescriptor.java index 687f517d99..e159a84cb1 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginDescriptor.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginDescriptor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,7 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; /** - * Describes the {@link VertexManagerPlugin} + * Describes the {@link VertexManagerPlugin} */ @Public public class VertexManagerPluginDescriptor extends EntityDescriptor { @@ -31,7 +31,7 @@ public class VertexManagerPluginDescriptor extends EntityDescriptor + * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.api.client; @@ -71,7 +71,7 @@ public abstract DAGStatus getDAGStatus(@Nullable Set statusOption * @throws TezException */ public abstract DAGStatus getDAGStatus(@Nullable Set statusOptions, - long timeout) + long timeout) throws IOException, TezException; /** @@ -80,8 +80,8 @@ public abstract DAGStatus getDAGStatus(@Nullable Set statusOption * specified options */ public abstract VertexStatus getVertexStatus(String vertexName, - Set statusOptions) - throws IOException, TezException; + Set statusOptions) + throws IOException, TezException; /** * Get the dag identifier for the currently executing dag. This is a string @@ -104,11 +104,11 @@ public abstract VertexStatus getVertexStatus(String vertexName, /** * Wait forever for DAG to complete without printing any vertex statuses - * + * * @return Final DAG Status * @throws IOException * @throws TezException - * @throws InterruptedException + * @throws InterruptedException */ public abstract DAGStatus waitForCompletion() throws IOException, TezException, InterruptedException; @@ -128,14 +128,14 @@ public DAGStatus waitForCompletion(long timeMs) throws IOException, TezException /** * Wait for DAG to complete and periodically print *all* vertices' status. - * + * * @param statusGetOpts * : status get options. For example, to get counter pass * EnumSet.of(StatusGetOpts.GET_COUNTERS) * @return Final DAG Status * @throws IOException * @throws TezException - * @throws InterruptedException + * @throws InterruptedException */ public abstract DAGStatus waitForCompletionWithStatusUpdates(@Nullable Set statusGetOpts) throws IOException, TezException, InterruptedException; @@ -148,5 +148,4 @@ public abstract DAGStatus waitForCompletionWithStatusUpdates(@Nullable Set + * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -83,7 +83,7 @@ public class DAGClientImpl extends DAGClient { private boolean cleanupFrameworkClient; public DAGClientImpl(ApplicationId appId, String dagId, TezConfiguration conf, - @Nullable FrameworkClient frameworkClient, UserGroupInformation ugi) { + @Nullable FrameworkClient frameworkClient, UserGroupInformation ugi) { this.appId = appId; this.dagId = dagId; this.conf = conf; @@ -107,7 +107,7 @@ public DAGClientImpl(ApplicationId appId, String dagId, TezConfiguration conf, statusPollInterval = conf.getLong( TezConfiguration.TEZ_DAG_STATUS_POLLINTERVAL_MS, TezConfiguration.TEZ_DAG_STATUS_POLLINTERVAL_MS_DEFAULT); - if(statusPollInterval < 0) { + if (statusPollInterval < 0) { LOG.error("DAG Status poll interval cannot be negative and setting to default value."); statusPollInterval = TezConfiguration.TEZ_DAG_STATUS_POLLINTERVAL_MS_DEFAULT; } @@ -150,7 +150,7 @@ protected ApplicationReport getApplicationReportInternal() { @Override public DAGStatus getDAGStatus(@Nullable Set statusOptions, - final long timeout) throws TezException, IOException { + final long timeout) throws TezException, IOException { Preconditions.checkArgument(timeout >= -1, "Timeout must be >= -1"); // Short circuit a timeout of 0. @@ -192,7 +192,7 @@ public DAGStatus getDAGStatus(@Nullable Set statusOptions, } else { // From the RM. Fall through to the Sleep. } - } else if(dagStatus.getState() == DAGStatus.State.SUCCEEDED + } else if (dagStatus.getState() == DAGStatus.State.SUCCEEDED || dagStatus.getState() == DAGStatus.State.FAILED || dagStatus.getState() == DAGStatus.State.KILLED || dagStatus.getState() == DAGStatus.State.ERROR) { @@ -234,7 +234,7 @@ public DAGStatus getDAGStatus(@Nullable Set statusOptions, } protected DAGStatus getDAGStatusInternal(@Nullable Set statusOptions, - long timeout) throws TezException, IOException { + long timeout) throws TezException, IOException { if (!dagCompleted) { // fetch from AM. on Error and while DAG is still not completed (could not reach AM, AM got @@ -394,7 +394,7 @@ public void close() throws IOException { * @throws IOException */ private DAGStatus getDAGStatusViaAM(@Nullable Set statusOptions, - long timeout) throws IOException { + long timeout) throws IOException { DAGStatus dagStatus = null; try { dagStatus = realClient.getDAGStatus(statusOptions, timeout); @@ -462,7 +462,7 @@ protected DAGStatus getDAGStatusViaRM() throws TezException, IOException { throw new TezException(e); } - if(appReport == null) { + if (appReport == null) { throw new TezException("Unknown/Invalid appId: " + appId); } @@ -486,7 +486,7 @@ protected DAGStatus getDAGStatusViaRM() throws TezException, IOException { dagState = DAGProtos.DAGStatusStateProto.DAG_KILLED; break; case FINISHED: - switch(appReport.getFinalApplicationStatus()) { + switch (appReport.getFinalApplicationStatus()) { case UNDEFINED: case FAILED: dagState = DAGProtos.DAGStatusStateProto.DAG_FAILED; @@ -514,7 +514,7 @@ protected DAGStatus getDAGStatusViaRM() throws TezException, IOException { if (appReport.getFinalApplicationStatus() == FinalApplicationStatus.FAILED || appReport.getFinalApplicationStatus() == FinalApplicationStatus.KILLED) { long startTime = System.currentTimeMillis(); - while((appReport.getDiagnostics() == null + while ((appReport.getDiagnostics() == null || appReport.getDiagnostics().isEmpty()) && (System.currentTimeMillis() - startTime) < diagnoticsWaitTimeout) { try { @@ -598,7 +598,7 @@ private double monitorProgress(Set vertexNames, double prevDagProgress, dagProgress = getProgress(progress); boolean progressChanged = dagProgress > prevDagProgress; long currentTimeMillis = System.currentTimeMillis(); - long timeSinceLastPrintStatus = currentTimeMillis - lastPrintStatusTimeMillis; + long timeSinceLastPrintStatus = currentTimeMillis - lastPrintStatusTimeMillis; boolean printIntervalExpired = timeSinceLastPrintStatus > PRINT_STATUS_INTERVAL_MILLIS; if (progressChanged || printIntervalExpired) { lastPrintStatusTimeMillis = currentTimeMillis; diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImplLocal.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImplLocal.java index a0509cdc5f..fe8c8c0d4a 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImplLocal.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImplLocal.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -38,8 +38,8 @@ public class DAGClientImplLocal extends DAGClientImpl { private BiFunction, Long, DAGStatus> dagStatusFunction; public DAGClientImplLocal(ApplicationId appId, String dagId, TezConfiguration conf, - FrameworkClient frameworkClient, UserGroupInformation ugi, - BiFunction, Long, DAGStatus> dagStatusFunction) { + FrameworkClient frameworkClient, UserGroupInformation ugi, + BiFunction, Long, DAGStatus> dagStatusFunction) { super(appId, dagId, conf, frameworkClient, ugi); this.dagStatusFunction = dagStatusFunction; } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientInternal.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientInternal.java index 8346d53da7..7ac0205295 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientInternal.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientInternal.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.api.client; @@ -71,7 +71,7 @@ public abstract DAGStatus getDAGStatus(@Nullable Set statusOption * @throws TezException */ public abstract DAGStatus getDAGStatus(@Nullable Set statusOptions, - long timeout) + long timeout) throws IOException, TezException, ApplicationNotFoundException; /** @@ -80,8 +80,8 @@ public abstract DAGStatus getDAGStatus(@Nullable Set statusOption * specified options */ public abstract VertexStatus getVertexStatus(String vertexName, - Set statusOptions) - throws IOException, TezException, ApplicationNotFoundException; + Set statusOptions) + throws IOException, TezException, ApplicationNotFoundException; /** * Get the dag identifier for the currently executing dag. This is a string @@ -104,24 +104,24 @@ public abstract VertexStatus getVertexStatus(String vertexName, /** * Wait for DAG to complete without printing any vertex statuses - * + * * @return Final DAG Status * @throws IOException * @throws TezException - * @throws InterruptedException + * @throws InterruptedException */ public abstract DAGStatus waitForCompletion() throws IOException, TezException, InterruptedException; /** * Wait for DAG to complete and periodically print *all* vertices' status. - * + * * @param statusGetOpts * : status get options. For example, to get counter pass * EnumSet.of(StatusGetOpts.GET_COUNTERS) * @return Final DAG Status * @throws IOException * @throws TezException - * @throws InterruptedException + * @throws InterruptedException */ public abstract DAGStatus waitForCompletionWithStatusUpdates(@Nullable Set statusGetOpts) throws IOException, TezException, InterruptedException; diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java index 4ec9c94354..64b760c3f2 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.api.client; @@ -62,7 +62,6 @@ import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; - @Private public class DAGClientTimelineImpl extends DAGClientInternal { private static final Logger LOG = LoggerFactory.getLogger(DAGClientTimelineImpl.class); @@ -207,7 +206,7 @@ public DAGStatus waitForCompletionWithStatusUpdates( return getDAGStatus(statusGetOpts); } - @Override + @Override public void close() throws IOException { if (httpClient != null) { httpClient.destroy(); @@ -469,7 +468,7 @@ private boolean webappHttpsOnly(Configuration conf) throws TezException { try { Class yarnConfiguration = Class.forName("org.apache.hadoop.yarn.conf.YarnConfiguration"); final Method useHttps = yarnConfiguration.getMethod("useHttps", Configuration.class); - return (Boolean)useHttps.invoke(null, conf); + return (Boolean) useHttps.invoke(null, conf); } catch (ClassNotFoundException e) { throw new TezException(e); } catch (InvocationTargetException e) { @@ -499,7 +498,7 @@ protected Client getCachedHttpClient() throws IOException { put("ERROR", DAGStatusStateProto.DAG_ERROR); put("TERMINATING", DAGStatusStateProto.DAG_TERMINATING); put("COMMITTING", DAGStatusStateProto.DAG_COMMITTING); - }}); + }}); private static final Map vertexStateProtoMap = Collections.unmodifiableMap(new HashMap() {{ @@ -516,10 +515,9 @@ protected Client getCachedHttpClient() throws IOException { put("COMMITTING", VertexStatusStateProto.VERTEX_COMMITTING); }}); - @Override public DAGStatus getDAGStatus(@Nullable Set statusOptions, - long timeout) throws IOException, TezException { + long timeout) throws IOException, TezException { return getDAGStatus(statusOptions); } @@ -527,5 +525,4 @@ public DAGStatus getDAGStatus(@Nullable Set statusOptions, public String getWebUIAddress() throws IOException, TezException { throw new TezException("DAGClientTimelineImpl.getWebUIAddress is not supported"); } - } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java index 1f8db62b80..8a25d9192e 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGStatus.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.api.client; @@ -40,7 +40,7 @@ public class DAGStatus { private static final String LINE_SEPARATOR = System - .getProperty("line.separator"); + .getProperty("line.separator"); public enum State { SUBMITTED, // Returned from the RM only @@ -67,35 +67,35 @@ public DAGStatus(DAGStatusProtoOrBuilder proxy, DagStatusSource source) { } public State getState() { - switch(proxy.getState()) { - case DAG_SUBMITTED: - return DAGStatus.State.SUBMITTED; - // For simplicity, initing/terminating states are presented as running - case DAG_INITING: - case DAG_TERMINATING: - case DAG_RUNNING: - case DAG_COMMITTING: - return DAGStatus.State.RUNNING; - case DAG_SUCCEEDED: - return DAGStatus.State.SUCCEEDED; - case DAG_FAILED: - return DAGStatus.State.FAILED; - case DAG_KILLED: - return DAGStatus.State.KILLED; - case DAG_ERROR: - return DAGStatus.State.ERROR; - default: - throw new TezUncheckedException("Unsupported value for DAGStatus.State : " + - proxy.getState()); + switch (proxy.getState()) { + case DAG_SUBMITTED: + return DAGStatus.State.SUBMITTED; + // For simplicity, initing/terminating states are presented as running + case DAG_INITING: + case DAG_TERMINATING: + case DAG_RUNNING: + case DAG_COMMITTING: + return DAGStatus.State.RUNNING; + case DAG_SUCCEEDED: + return DAGStatus.State.SUCCEEDED; + case DAG_FAILED: + return DAGStatus.State.FAILED; + case DAG_KILLED: + return DAGStatus.State.KILLED; + case DAG_ERROR: + return DAGStatus.State.ERROR; + default: + throw new TezUncheckedException("Unsupported value for DAGStatus.State : " + + proxy.getState()); } } public boolean isCompleted() { State state = getState(); return (state == State.SUCCEEDED || - state == State.FAILED || - state == State.KILLED || - state == State.ERROR); + state == State.FAILED || + state == State.KILLED || + state == State.ERROR); } public List getDiagnostics() { @@ -112,7 +112,7 @@ public List getDiagnostics() { * @see Progress */ public Progress getDAGProgress() { - if(progress == null && proxy.hasDAGProgress()) { + if (progress == null && proxy.hasDAGProgress()) { progress = new Progress(proxy.getDAGProgress()); } return progress; @@ -128,11 +128,11 @@ public Progress getDAGProgress() { * @see Progress */ public Map getVertexProgress() { - if(vertexProgress == null) { - if(proxy.getVertexProgressList() != null) { + if (vertexProgress == null) { + if (proxy.getVertexProgressList() != null) { List kvList = proxy.getVertexProgressList(); vertexProgress = new LinkedHashMap(kvList.size()); - for(StringProgressPairProto kv : kvList){ + for (StringProgressPairProto kv : kvList) { vertexProgress.put(kv.getKey(), new Progress(kv.getProgress())); } } @@ -146,7 +146,7 @@ public TezCounters getDAGCounters() { } if (proxy.hasDagCounters()) { dagCounters = DagTypeConverters.convertTezCountersFromProto( - proxy.getDagCounters()); + proxy.getDagCounters()); } countersInitialized.set(true); return dagCounters; @@ -168,15 +168,15 @@ DagStatusSource getSource() { @Override public boolean equals(Object obj) { // Source explicitly exclude from equals - if (obj instanceof DAGStatus){ - DAGStatus other = (DAGStatus)obj; + if (obj instanceof DAGStatus) { + DAGStatus other = (DAGStatus) obj; return getState() == other.getState() && getDiagnostics().equals(other.getDiagnostics()) && getDAGProgress().equals(other.getDAGProgress()) && getVertexProgress().equals(other.getVertexProgress()) - && - ((getDAGCounters() == null && other.getDAGCounters() == null) - || getDAGCounters().equals(other.getDAGCounters())); + && + ((getDAGCounters() == null && other.getDAGCounters() == null) + || getDAGCounters().equals(other.getDAGCounters())); } return false; } @@ -195,13 +195,13 @@ public int hashCode() { TezCounters counters = getDAGCounters(); result = prime * result + - ((diagnostics == null)? 0 : diagnostics.hashCode()); + ((diagnostics == null) ? 0 : diagnostics.hashCode()); result = prime * result + - ((dagProgress == null)? 0 : dagProgress.hashCode()); + ((dagProgress == null) ? 0 : dagProgress.hashCode()); result = prime * result + - ((vProgress == null)? 0 : vProgress.hashCode()); + ((vProgress == null) ? 0 : vProgress.hashCode()); result = prime * result + - ((counters == null)? 0 : counters.hashCode()); + ((counters == null) ? 0 : counters.hashCode()); return result; } @@ -217,5 +217,4 @@ public String toString() { sb.append(", counters=" + (getDAGCounters() == null ? "null" : getDAGCounters().toString())); return sb.toString(); } - } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/Progress.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/Progress.java index 656838dc5a..1a157524bf 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/Progress.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/Progress.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.api.client; @@ -28,13 +28,13 @@ @Public @Evolving public class Progress { - + ProgressProtoOrBuilder proxy = null; - + Progress(ProgressProtoOrBuilder proxy) { this.proxy = proxy; } - + public int getTotalTaskCount() { return proxy.getTotalTaskCount(); } @@ -69,9 +69,9 @@ public int getRejectedTaskAttemptCount() { @Override public boolean equals(Object obj) { - if (obj instanceof Progress){ - Progress other = (Progress)obj; - return getTotalTaskCount() == other.getTotalTaskCount() + if (obj instanceof Progress) { + Progress other = (Progress) obj; + return getTotalTaskCount() == other.getTotalTaskCount() && getSucceededTaskCount() == other.getSucceededTaskCount() && getRunningTaskCount() == other.getRunningTaskCount() && getFailedTaskCount() == other.getFailedTaskCount() @@ -114,9 +114,9 @@ public String toString() { sb.append(getSucceededTaskCount()); sb.append(" Running: "); sb.append(getRunningTaskCount()); - sb.append(" Failed: "); + sb.append(" Failed: "); sb.append(getFailedTaskCount()); - sb.append(" Killed: "); + sb.append(" Killed: "); sb.append(getKilledTaskCount()); if (getFailedTaskAttemptCount() > 0) { sb.append(" FailedTaskAttempts: "); @@ -132,5 +132,4 @@ public String toString() { } return sb.toString(); } - } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/StatusGetOpts.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/StatusGetOpts.java index 3518d33c2c..57bef8d973 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/StatusGetOpts.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/StatusGetOpts.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/TimelineReaderFactory.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/TimelineReaderFactory.java index fc1595fa49..35e5c4c9c4 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/TimelineReaderFactory.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/TimelineReaderFactory.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -102,9 +102,9 @@ public static TimelineReaderStrategy getTimelineReaderStrategy(Configuration con } private static TimelineReaderStrategy getTimelineReaderStrategy(boolean isTokenDelegationSupported, - Configuration conf, - boolean useHttps, - int connTimeout) { + Configuration conf, + boolean useHttps, + int connTimeout) { TimelineReaderStrategy timelineReaderStrategy; if (isTokenDelegationSupported) { @@ -133,6 +133,7 @@ public static boolean isTimelineClientSupported() { public interface TimelineReaderStrategy { Client getHttpClient() throws IOException; + void close(); } @@ -227,14 +228,14 @@ public TokenAuthenticatedURLConnectionFactory(ConnectionConfigurator connConfigu @Override public HttpURLConnection getHttpURLConnection(URL url) throws IOException { try { - AuthenticatedURL authenticatedURL= ReflectionUtils.createClazzInstance( - DELEGATION_TOKEN_AUTHENTICATED_URL_CLAZZ_NAME, new Class[] { - delegationTokenAuthenticatorClazz, - ConnectionConfigurator.class - }, new Object[] { - authenticator, - connConfigurator - }); + AuthenticatedURL authenticatedURL = ReflectionUtils.createClazzInstance( + DELEGATION_TOKEN_AUTHENTICATED_URL_CLAZZ_NAME, new Class[]{ + delegationTokenAuthenticatorClazz, + ConnectionConfigurator.class + }, new Object[]{ + authenticator, + connConfigurator + }); return ReflectionUtils.invokeMethod(authenticatedURL, delegationTokenAuthenticateURLOpenConnectionMethod, url, token, doAsUser); } catch (Exception e) { @@ -397,7 +398,6 @@ private static synchronized boolean tokenDelegationSupported() { URL.class, delegationTokenAuthenticatedURLTokenClazz, String.class); isTokenDelegationClassesPresent = true; - } catch (TezException e) { LOG.info("Could not find class required for token delegation, will fallback to pseudo auth"); } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java index 9efb12d612..1c15b4204f 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/VertexStatus.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.api.client; @@ -69,7 +69,7 @@ public State getState() { @VisibleForTesting static State getState(DAGProtos.VertexStatusStateProto stateProto) { - switch(stateProto) { + switch (stateProto) { case VERTEX_NEW: return VertexStatus.State.NEW; case VERTEX_INITIALIZING: @@ -101,7 +101,7 @@ public List getDiagnostics() { } public Progress getProgress() { - if(progress == null && proxy.hasProgress()) { + if (progress == null && proxy.hasProgress()) { progress = new Progress(proxy.getProgress()); } return progress; @@ -113,21 +113,21 @@ public TezCounters getVertexCounters() { } if (proxy.hasVertexCounters()) { vertexCounters = DagTypeConverters.convertTezCountersFromProto( - proxy.getVertexCounters()); + proxy.getVertexCounters()); } countersInitialized.set(true); return vertexCounters; } - + @Override public boolean equals(Object obj) { - if (obj instanceof VertexStatus){ - VertexStatus other = (VertexStatus)obj; + if (obj instanceof VertexStatus) { + VertexStatus other = (VertexStatus) obj; return getState().equals(other.getState()) && getDiagnostics().equals(other.getDiagnostics()) && getProgress().equals(other.getProgress()) - && - ((getVertexCounters() == null && other.getVertexCounters() == null) + && + ((getVertexCounters() == null && other.getVertexCounters() == null) || getVertexCounters().equals(other.getVertexCounters())); } return false; @@ -145,11 +145,11 @@ public int hashCode() { TezCounters counters = getVertexCounters(); result = prime * result + - ((diagnostics == null)? 0 : diagnostics.hashCode()); + ((diagnostics == null) ? 0 : diagnostics.hashCode()); result = prime * result + - ((vProgress == null)? 0 : vProgress.hashCode()); + ((vProgress == null) ? 0 : vProgress.hashCode()); result = prime * result + - ((counters == null)? 0 : counters.hashCode()); + ((counters == null) ? 0 : counters.hashCode()); return result; } @@ -158,10 +158,9 @@ public int hashCode() { public String toString() { StringBuilder sb = new StringBuilder(); sb.append("status=" + getState() - + ", progress=" + getProgress() - + ", counters=" - + (getVertexCounters() == null ? "null" : getVertexCounters().toString())); + + ", progress=" + getProgress() + + ", counters=" + + (getVertexCounters() == null ? "null" : getVertexCounters().toString())); return sb.toString(); } - } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPB.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPB.java index 9d7ebeefee..c06aa2e84c 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPB.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPB.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,7 +26,7 @@ @ProtocolInfo( protocolName = "org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolBlockingPB", protocolVersion = 1) -public interface DAGClientAMProtocolBlockingPB - extends DAGClientAMProtocol.BlockingInterface { +public interface DAGClientAMProtocolBlockingPB + extends DAGClientAMProtocol.BlockingInterface { } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java index 798160a4ab..3b881915b2 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -73,7 +73,7 @@ public class DAGClientRPCImpl extends DAGClientInternal { private UserGroupInformation ugi; public DAGClientRPCImpl(ApplicationId appId, String dagId, - TezConfiguration conf, @Nullable FrameworkClient frameworkClient, UserGroupInformation ugi) { + TezConfiguration conf, @Nullable FrameworkClient frameworkClient, UserGroupInformation ugi) { this.appId = appId; this.dagId = dagId; this.conf = conf; @@ -93,10 +93,9 @@ public DAGStatus getDAGStatus(Set statusOptions) return getDAGStatus(statusOptions, 0); } - @Override public DAGStatus getDAGStatus(@Nullable Set statusOptions, - long timeout) throws IOException, TezException, ApplicationNotFoundException { + long timeout) throws IOException, TezException, ApplicationNotFoundException { if (createAMProxyIfNeeded()) { try { DAGStatus dagStatus = getDAGStatusViaAM(statusOptions, timeout); @@ -116,10 +115,10 @@ public DAGStatus getDAGStatus(@Nullable Set statusOptions, @Override public VertexStatus getVertexStatus(String vertexName, - Set statusOptions) + Set statusOptions) throws IOException, TezException, ApplicationNotFoundException { - if(createAMProxyIfNeeded()) { + if (createAMProxyIfNeeded()) { try { return getVertexStatusViaAM(vertexName, statusOptions); } catch (TezException e) { @@ -144,7 +143,6 @@ public String getSessionIdentifierString() { return appId.toString(); } - @Override public void tryKillDAG() throws TezException, IOException { LOG.debug("TryKill for app: {} dag:{}", appId, dagId); @@ -176,7 +174,7 @@ protected ApplicationReport getApplicationReportInternal() { } void resetProxy(Exception e) { - if(LOG.isDebugEnabled()) { + if (LOG.isDebugEnabled()) { LOG.debug("Resetting AM proxy for app: " + appId + " dag:" + dagId + " due to exception :", e); } @@ -188,17 +186,17 @@ DAGStatus getDAGStatusViaAM(Set statusOptions, long timeout) LOG.debug("GetDAGStatus via AM for app: {} dag:{}", appId, dagId); GetDAGStatusRequestProto.Builder requestProtoBuilder = GetDAGStatusRequestProto.newBuilder() - .setDagId(dagId).setTimeout(timeout); + .setDagId(dagId).setTimeout(timeout); if (statusOptions != null) { requestProtoBuilder.addAllStatusOptions( - DagTypeConverters.convertStatusGetOptsToProto(statusOptions)); + DagTypeConverters.convertStatusGetOptsToProto(statusOptions)); } try { return new DAGStatus( - proxy.getDAGStatus(null, - requestProtoBuilder.build()).getDagStatus(), DagStatusSource.AM); + proxy.getDAGStatus(null, + requestProtoBuilder.build()).getDagStatus(), DagStatusSource.AM); } catch (ServiceException e) { RPCUtil.unwrapAndThrowException(e); // Should not reach here @@ -207,7 +205,7 @@ DAGStatus getDAGStatusViaAM(Set statusOptions, long timeout) } VertexStatus getVertexStatusViaAM(String vertexName, - Set statusOptions) + Set statusOptions) throws TezException, IOException { if (LOG.isDebugEnabled()) { LOG.debug("GetVertexStatus via AM for app: " + appId + " dag: " + dagId @@ -215,18 +213,18 @@ VertexStatus getVertexStatusViaAM(String vertexName, } GetVertexStatusRequestProto.Builder requestProtoBuilder = GetVertexStatusRequestProto.newBuilder() - .setDagId(dagId) - .setVertexName(vertexName); + .setDagId(dagId) + .setVertexName(vertexName); if (statusOptions != null) { requestProtoBuilder.addAllStatusOptions( - DagTypeConverters.convertStatusGetOptsToProto(statusOptions)); + DagTypeConverters.convertStatusGetOptsToProto(statusOptions)); } try { return new VertexStatus( - proxy.getVertexStatus(null, - requestProtoBuilder.build()).getVertexStatus()); + proxy.getVertexStatus(null, + requestProtoBuilder.build()).getVertexStatus()); } catch (ServiceException e) { RPCUtil.unwrapAndThrowException(e); // Should not reach here @@ -263,25 +261,25 @@ ApplicationReport getAppReport() throws IOException, TezException, boolean createAMProxyIfNeeded() throws IOException, TezException, ApplicationNotFoundException { - if(proxy != null) { + if (proxy != null) { // if proxy exist optimistically use it assuming there is no retry return true; } appReport = null; appReport = getAppReport(); - if(appReport == null) { + if (appReport == null) { return false; } YarnApplicationState appState = appReport.getYarnApplicationState(); - if(appState != YarnApplicationState.RUNNING) { + if (appState != YarnApplicationState.RUNNING) { return false; } // YARN-808. Cannot ascertain if AM is ready until we connect to it. // workaround check the default string set by YARN - if(appReport.getHost() == null || appReport.getHost().equals("N/A") || - appReport.getRpcPort() == 0){ + if (appReport.getHost() == null || appReport.getHost().equals("N/A") || + appReport.getRpcPort() == 0) { // attempt not running return false; } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/package-info.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/package-info.java index 3c18e02a5b..d5cdc0131f 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/package-info.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/package-info.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,4 +19,4 @@ @Private package org.apache.tez.dag.api.client.rpc; -import org.apache.hadoop.classification.InterfaceAudience.Private; \ No newline at end of file +import org.apache.hadoop.classification.InterfaceAudience.Private; diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/event/VertexStateUpdate.java b/tez-api/src/main/java/org/apache/tez/dag/api/event/VertexStateUpdate.java index 5b7ca40b1f..59c0fcda68 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/event/VertexStateUpdate.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/event/VertexStateUpdate.java @@ -31,7 +31,6 @@ public class VertexStateUpdate { private final String vertexName; private final VertexState vertexState; - public VertexStateUpdate(String vertexName, VertexState vertexState) { this.vertexName = vertexName; this.vertexState = vertexState; @@ -39,6 +38,7 @@ public VertexStateUpdate(String vertexName, VertexState vertexState) { /** * Get the name of the vertex for which the state has changed + * * @return the name of the vertex */ public String getVertexName() { @@ -47,6 +47,7 @@ public String getVertexName() { /** * Get the updated state + * * @return the updated state */ public VertexState getVertexState() { diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/event/VertexStateUpdateParallelismUpdated.java b/tez-api/src/main/java/org/apache/tez/dag/api/event/VertexStateUpdateParallelismUpdated.java index 54fa87cdda..8d8444c171 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/event/VertexStateUpdateParallelismUpdated.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/event/VertexStateUpdateParallelismUpdated.java @@ -25,6 +25,7 @@ public class VertexStateUpdateParallelismUpdated extends VertexStateUpdate { private final int parallelism; private final int previousParallelism; + public VertexStateUpdateParallelismUpdated(String vertexName, int updatedParallelism, int previousParallelism) { super(vertexName, VertexState.PARALLELISM_UPDATED); @@ -34,6 +35,7 @@ public VertexStateUpdateParallelismUpdated(String vertexName, /** * Returns the new parallelism for the vertex + * * @return the new parallelism */ public int getParallelism() { @@ -42,6 +44,7 @@ public int getParallelism() { /** * Returns the previous value of the parallelism + * * @return the previous parallelism */ public int getPreviousParallelism() { diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalIOProcessor.java b/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalIOProcessor.java index 5a4cbe8113..42d9d2af94 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalIOProcessor.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalIOProcessor.java @@ -21,9 +21,8 @@ /** * Abstract representation of the interface {@link LogicalIOProcessor}. - * Implements the base logic of some methods into this class and is expected + * Implements the base logic of some methods into this class and is expected * to be the base class that is derived to implement the user {@link Processor} - * */ @Public public abstract class AbstractLogicalIOProcessor implements LogicalIOProcessor, @@ -52,5 +51,4 @@ public final ProcessorContext getContext() { @Override public void abort() { } - } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalInput.java b/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalInput.java index a97f3fa6d4..f54d870ea4 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalInput.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalInput.java @@ -24,10 +24,10 @@ /** * An abstract class which should be the base class for all implementations of LogicalInput. - * + *

* This class implements the framework facing as well as user facing methods which need to be * implemented by all LogicalInputs. - * + *

* This includes default implementations of a new method for convenience. * * Input classes must provide a 2 argument public constructor for Tez to create the @@ -36,7 +36,6 @@ * setup the number of physical inputs that the logical input will see. * Tez will take care of initializing and closing the Input after a {@link Processor} completes.

*

- * */ @Public public abstract class AbstractLogicalInput implements LogicalInput, LogicalInputFrameworkInterface { diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalOutput.java b/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalOutput.java index f36b7f2ec9..0c981ab07a 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalOutput.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/AbstractLogicalOutput.java @@ -23,12 +23,11 @@ /** * An abstract class which should be the base class for all implementations of LogicalOutput. - * + *

* This class implements the framework facing as well as user facing methods which need to be * implemented by all LogicalOutputs. - * + *

* This includes default implementations of a new method for convenience. - * */ @Public public abstract class AbstractLogicalOutput implements LogicalOutput, LogicalOutputFrameworkInterface { diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/DagIdentifier.java b/tez-api/src/main/java/org/apache/tez/runtime/api/DagIdentifier.java index dd63b4cfa1..4f26418eab 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/DagIdentifier.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/DagIdentifier.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,6 +21,6 @@ public interface DagIdentifier { public String getName(); - + public int getIdentifier(); } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/Event.java b/tez-api/src/main/java/org/apache/tez/runtime/api/Event.java index d9c00d3b02..8495617141 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/Event.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/Event.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/ExecutionContext.java b/tez-api/src/main/java/org/apache/tez/runtime/api/ExecutionContext.java index fa9a47ff0d..b8b7664dfc 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/ExecutionContext.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/ExecutionContext.java @@ -18,7 +18,7 @@ /** * The context for the executor within which a task runs. May be shared between tasks - * + *

* This interface is not meant to be implemented by users */ @InterfaceAudience.Public @@ -26,6 +26,7 @@ public interface ExecutionContext { /** * Get the hostname on which the JVM is running. + * * @return the hostname */ public String getHostName(); diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/Input.java b/tez-api/src/main/java/org/apache/tez/runtime/api/Input.java index bda439616e..c0e0b90fa5 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/Input.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/Input.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,27 +26,26 @@ * * This interface has methods which can be used by a {@link org.apache.tez.runtime.api.Processor} * to control execution of this Input and read data from it. - * + * * Actual implementations are expected to derive from {@link AbstractLogicalInput} */ @Public public interface Input { - /** * Start any processing that the Input may need to perform. It is the * responsibility of the Processor to start Inputs. - * + * * This typically acts as a signal to Inputs to start any Processing that they * may required. A blocking implementation of this method should not be used * as a mechanism to determine when an Input is actually ready. - * + * * This method may be invoked by the framework under certain circumstances, * and as such requires the implementation to be non-blocking. - * + * * Inputs must be written to handle multiple start invocations - typically * honoring only the first one. - * + * * @throws Exception */ public void start() throws Exception; diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/InputContext.java b/tez-api/src/main/java/org/apache/tez/runtime/api/InputContext.java index 6eac2dfc95..de26c17b28 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/InputContext.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/InputContext.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -38,21 +38,21 @@ public interface InputContext extends TaskContext { * @return the convenient string */ String getInputOutputVertexNames(); - + /** * Get the index of the input in the set of all inputs for the task. The * index will be consistent and valid only among the tasks of this vertex. * @return index */ public int getInputIndex(); - + /** * Inform the framework that the specific Input is ready for consumption. - * + * * This method can be invoked multiple times. */ public void inputIsReady(); - + /** * Get an {@link InputStatisticsReporter} for this {@link Input} that can * be used to report statistics like data size diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/InputFrameworkInterface.java b/tez-api/src/main/java/org/apache/tez/runtime/api/InputFrameworkInterface.java index 7f0a1d995f..d820ceb0ef 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/InputFrameworkInterface.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/InputFrameworkInterface.java @@ -24,24 +24,23 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; - /** * Represents an input through which a TezProcessor receives data on an edge. *

- * + *

* This interface has methods which are used by the Tez framework to control the Input. *

- * + *

* During initialization, Inputs must specify an initial memory requirement via * {@link InputContext}.requestInitialMemory *

- * - * + *

+ *

* Inputs must also inform the framework once they are ready to be consumed. * This typically means that the Processor will not block when reading from the * corresponding Input. This is done via {@link InputContext}.inputIsReady. * Inputs choose the policy on when they are ready. - * + *

* Input implementations are expected to derive from {@link AbstractLogicalInput} */ @Public @@ -50,8 +49,7 @@ public interface InputFrameworkInterface { * Initializes the Input. * * @return list of events that were generated during initialization - * @throws Exception - * if an error occurs + * @throws Exception if an error occurs */ public List initialize() throws Exception; @@ -59,19 +57,15 @@ public interface InputFrameworkInterface { * Handles user and system generated {@link Event}s, which typically carry * information such as an output being available on the previous vertex. * - * @param inputEvents - * the list of {@link Event}s + * @param inputEvents the list of {@link Event}s */ public void handleEvents(List inputEvents) throws Exception; - - /** * Closes the Input * * @return list of events that were generated during close - * @throws Exception - * if an error occurs + * @throws Exception if an error occurs */ public List close() throws Exception; } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializer.java b/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializer.java index cc33205e3a..7fab34cb77 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializer.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializer.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -57,7 +57,7 @@ public InputInitializer(InputInitializerContext initializerContext) { * processing the event being done via this method. The moment this method * returns a list of events, input initialization is considered to be * complete. - * + * * @return a list of events which are eventually routed to a * {@link org.apache.tez.dag.api.VertexManagerPlugin} for routing * @throws Exception @@ -80,7 +80,7 @@ public abstract void handleInputInitializerEvent(List eve /** * Return ahe {@link org.apache.tez.runtime.api.InputInitializerContext} * for this specific instance of the Initializer. - * + * * @return the {@link org.apache.tez.runtime.api.InputInitializerContext} * for the initializer */ diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializerContext.java b/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializerContext.java index 7c9562e7c9..a2958ad512 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializerContext.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/InputInitializerContext.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -44,7 +44,7 @@ public interface InputInitializerContext { * @return Application id */ ApplicationId getApplicationId(); - + /** * Get the name of the DAG * @return DAG name @@ -68,26 +68,26 @@ public interface InputInitializerContext { * @return User payload */ UserPayload getInputUserPayload(); - + /** * Get the user payload for the initializer * @return User payload */ UserPayload getUserPayload(); - + /** * Get the number of tasks in this vertex. Maybe -1 if the vertex has not been * initialized with a pre-determined number of tasks. * @return number of tasks */ int getNumTasks(); - + /** * Get the resource allocated to a task of this vertex * @return Resource */ Resource getVertexTaskResource(); - + /** * Get the total resource allocated to this vertex. If the DAG is running in * a busy cluster then it may have no resources available dedicated to it. The @@ -95,7 +95,7 @@ public interface InputInitializerContext { * @return Resource */ Resource getTotalAvailableResource(); - + /** * Get the number of nodes in the cluster * @return Number of nodes diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/InputSpecUpdate.java b/tez-api/src/main/java/org/apache/tez/runtime/api/InputSpecUpdate.java index c6bbba7090..8e99f99cd7 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/InputSpecUpdate.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/InputSpecUpdate.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,7 +31,7 @@ * Update Input specs for Inputs running in a task. Allows setting the number of physical * inputs for all work units if they have the same number of physical inputs, or individual * numPhysicalInputs for each work unit. - * + * */ @Unstable @Public @@ -41,11 +41,11 @@ public class InputSpecUpdate { private final List numPhysicalInputs; private final static InputSpecUpdate DEFAULT_SINGLE_PHYSICAL_INPUT_SPEC = createAllTaskInputSpecUpdate(1); - + /** * Create an update instance where all work units (typically represented by * {@link InputDataInformationEvent}) will have the same number of physical inputs. - * + * * @param numPhysicalInputs * the number of physical inputs for all work units which will use the LogicalInput * @return a new InputSpecUpdate instance given the parameters @@ -57,7 +57,7 @@ public static InputSpecUpdate createAllTaskInputSpecUpdate(int numPhysicalInputs /** * Create an update instance where all work units (typically represented by * {@link InputDataInformationEvent}) will have the same number of physical inputs. - * + * * @param perWorkUnitNumPhysicalInputs * A list containing one entry per work unit. The order in the list corresponds to task * index or equivalently the order of {@link InputDataInformationEvent}s being sent. @@ -67,7 +67,7 @@ public static InputSpecUpdate createPerTaskInputSpecUpdate( List perWorkUnitNumPhysicalInputs) { return new InputSpecUpdate(perWorkUnitNumPhysicalInputs); } - + public static InputSpecUpdate getDefaultSinglePhysicalInputSpecUpdate() { return DEFAULT_SINGLE_PHYSICAL_INPUT_SPEC; } @@ -90,13 +90,13 @@ public int getNumPhysicalInputsForWorkUnit(int index) { return numPhysicalInputs.get(index); } } - + @Private /* Used for recovery serialization */ public boolean isForAllWorkUnits() { return this.forAllWorkUnits; } - + @Private /* Used for recovery serialization */ public List getAllNumPhysicalInputs() { diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatistics.java b/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatistics.java index 1066dbb861..5016fac957 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatistics.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatistics.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.runtime.api; @@ -30,20 +30,19 @@ @Public @Evolving public interface InputStatistics { - + /** * Returns the data size associated with this logical input *
It is the size of the data read from this input by the vertex. * @return Data size in bytes */ - public long getDataSize(); - + public long getDataSize(); + /** * Get the numbers of items processed. These could be key-value pairs, table * records etc. - * + * * @return Number of items processed */ - public long getItemsProcessed(); - + public long getItemsProcessed(); } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatisticsReporter.java b/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatisticsReporter.java index a85d25b9bd..df0833e2a2 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatisticsReporter.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/InputStatisticsReporter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -37,11 +37,9 @@ public interface InputStatisticsReporter { /** * Report the number of items processed. These could be key-value pairs, table * records etc. - * + * * @param items * number of items */ public void reportItemsProcessed(long items); - - } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessor.java b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessor.java index 0256d1ec02..b913e6e915 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessor.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessorFrameworkInterface.java b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessorFrameworkInterface.java index edae2f8b81..1a83d6fbed 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessorFrameworkInterface.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalIOProcessorFrameworkInterface.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -32,7 +32,7 @@ public interface LogicalIOProcessorFrameworkInterface extends ProcessorFramework /** * Runs the {@link Processor} - * + * * @param inputs * a map of the source vertex name to {@link LogicalInput} - one per * incoming edge. @@ -42,6 +42,5 @@ public interface LogicalIOProcessorFrameworkInterface extends ProcessorFramework * @throws Exception TODO */ public void run(Map inputs, - Map outputs) throws Exception; - + Map outputs) throws Exception; } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalInput.java b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalInput.java index 9f10d0bef5..c5242cad68 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalInput.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalInput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,7 +25,7 @@ * edge. A {@link LogicalIOProcessor} sees a single Logical Input per incoming * edge. It provides a unified view that hides the details of the actual physical * inputs - * + * * User implementations are expected to derive from {@link AbstractLogicalInput} */ @Public diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalInputFrameworkInterface.java b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalInputFrameworkInterface.java index 191dcd0784..d78c710cf0 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalInputFrameworkInterface.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalInputFrameworkInterface.java @@ -24,7 +24,7 @@ /** * Defines the framework facing interfact for a {@link LogicalInput} - * + *

* User implementations are expected to derive from {@link AbstractLogicalInput} */ @Public diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalOutput.java b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalOutput.java index 29221895d6..2f7f50994e 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalOutput.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/LogicalOutput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/MemoryUpdateCallback.java b/tez-api/src/main/java/org/apache/tez/runtime/api/MemoryUpdateCallback.java index 1f304b380f..e2fa0642ab 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/MemoryUpdateCallback.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/MemoryUpdateCallback.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,12 +26,11 @@ * memory. This will typically be implemented by authors of {@link Input}s, * {@link Output}s and potentially {@link Processor}s in order to request * for appropriate memory limits for the respective components. - * + * */ @Unstable @Public public abstract class MemoryUpdateCallback { public abstract void memoryAssigned(long assignedSize); - } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/MergedInputContext.java b/tez-api/src/main/java/org/apache/tez/runtime/api/MergedInputContext.java index 65bb087c7a..4a0a88fa73 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/MergedInputContext.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/MergedInputContext.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -33,19 +33,19 @@ public interface MergedInputContext { * @return {@link UserPayload} */ public UserPayload getUserPayload(); - + /** * Inform the framework that the specific Input is ready for consumption. - * + * * This method can be invoked multiple times. */ public void inputIsReady(); - + /** * Inform the framework that progress has been made */ public void notifyProgress(); - + /** * Get the work directories for the Input * @return an array of work dirs diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/MergedLogicalInput.java b/tez-api/src/main/java/org/apache/tez/runtime/api/MergedLogicalInput.java index e3c36244f8..8f67bb3f28 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/MergedLogicalInput.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/MergedLogicalInput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -42,7 +42,6 @@ @Evolving public abstract class MergedLogicalInput implements LogicalInput { - private AtomicBoolean notifiedInputReady = new AtomicBoolean(false); private List inputs; private final AtomicBoolean isStarted = new AtomicBoolean(false); @@ -66,11 +65,11 @@ public MergedLogicalInput(MergedInputContext context, List inputs) { public final List getInputs() { return inputs; } - + public final MergedInputContext getContext() { return context; } - + @Override public final void start() throws Exception { if (!isStarted.getAndSet(true)) { diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/ObjectRegistry.java b/tez-api/src/main/java/org/apache/tez/runtime/api/ObjectRegistry.java index a280942225..128a012dab 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/ObjectRegistry.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/ObjectRegistry.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -42,7 +42,7 @@ public interface ObjectRegistry { * as there is only one instance of an Object stored for a given key * irrespective of the life-cycle attached to the Object. The object may stay * in the cache while the Vertex (to which the task belongs) is running. - * + * * @param key * Key to identify the Object * @param value @@ -50,16 +50,16 @@ public interface ObjectRegistry { * @return Previous Object associated with the key attached if present else * null. Could return the same object if the object was associated * with the same key for a different life-cycle. - */ + */ public Object cacheForVertex(String key, Object value); - + /** * Insert or update object into the registry with DAG life-cycle. This will * remove an object associated with the same key with a different life-cycle * as there is only one instance of an Object stored for a given key * irrespective of the life-cycle attached to the Object. The object may stay * in the cache while the DAG (to which the task belongs) is running. - * + * * @param key * Key to identify the Object * @param value @@ -67,16 +67,16 @@ public interface ObjectRegistry { * @return Previous Object associated with the key attached if present else * null. Could return the same object if the object was associated * with the same key for a different life-cycle. - */ + */ public Object cacheForDAG(String key, Object value); - + /** * Insert or update object into the registry with Session life-cycle. This * will remove an object associated with the same key with a different * life-cycle as there is only one instance of an Object stored for a given * key irrespective of the life-cycle attached to the Object. The object may stay * in the cache while the Session (to which the task belongs) is running. - * + * * @param key * Key to identify the Object * @param value @@ -84,7 +84,7 @@ public interface ObjectRegistry { * @return Previous Object associated with the key attached if present else * null. Could return the same object if the object was associated * with the same key for a different life-cycle. - */ + */ public Object cacheForSession(String key, Object value); /** @@ -100,5 +100,4 @@ public interface ObjectRegistry { * @return True if an object was found and removed */ public boolean delete(String key); - } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/Output.java b/tez-api/src/main/java/org/apache/tez/runtime/api/Output.java index 9125c0d940..9ac0eb89af 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/Output.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/Output.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,28 +26,26 @@ * * This interface has methods which can be used by a {@link org.apache.tez.runtime.api.Processor} * to control execution of this Output and write data to it. - * + * * Users are expected to derive from {@link AbstractLogicalOutput} * */ @Public public interface Output { - - /** * Start any processing that the Output may need to perform. It is the * responsibility of the Processor to start Outputs. - * + * * This typically acts as a signal to Outputs to start any Processing that they * may required. - * + * * This method may be invoked by the framework under certain circumstances, * and as such requires the implementation to be non-blocking. - * + * * Outputs must be written to handle multiple start invocations - typically * honoring only the first one. - * + * * @throws Exception */ public void start() throws Exception; @@ -60,5 +58,4 @@ public interface Output { * if an error occurs */ public Writer getWriter() throws Exception; - -} \ No newline at end of file +} diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitter.java b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitter.java index 684772a172..0cb1b81200 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitter.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -67,7 +67,7 @@ public OutputCommitter(OutputCommitterContext committerContext) { * master process. Based on user configuration, commit is called at the end of * the DAG execution for all outputs or immediately upon completion of all the * tasks that produced the output. This is guaranteed to only be called once. - * + * * @throws java.lang.Exception */ public abstract void commitOutput() throws Exception; @@ -77,13 +77,13 @@ public OutputCommitter(OutputCommitterContext committerContext) { * non-successful state. This is also called to abort a previously committed * output in the case of a post-commit failure. This is called from the * application master process. This may be called multiple times. - * + * * @param finalState * final run-state of the vertex * @throws java.lang.Exception */ public abstract void abortOutput(VertexStatus.State finalState) - throws Exception; + throws Exception; /** * Whether the OutputCommitter supports recovery of output from a Task @@ -100,7 +100,7 @@ public boolean isTaskRecoverySupported() { * @param previousDAGAttempt Previous DAG Attempt Number * @throws java.lang.Exception */ - public void recoverTask(int taskIndex, int previousDAGAttempt) throws Exception { + public void recoverTask(int taskIndex, int previousDAGAttempt) throws Exception { } /** @@ -112,5 +112,4 @@ public void recoverTask(int taskIndex, int previousDAGAttempt) throws Exception public final OutputCommitterContext getContext() { return this.committerContext; } - } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitterContext.java b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitterContext.java index d254a6a99e..8bb053fb73 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitterContext.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputCommitterContext.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -71,11 +71,10 @@ public interface OutputCommitterContext { * @return User Payload */ public UserPayload getUserPayload(); - + /** * Get Vertex Index in the DAG * @return Vertex index */ public int getVertexIndex(); - } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputContext.java b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputContext.java index f0de897fda..9444415a39 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputContext.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputContext.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -47,7 +47,7 @@ public interface OutputContext extends TaskContext { * @return index */ int getOutputIndex(); - + /** * Get an {@link OutputStatisticsReporter} for this {@link Output} that can * be used to report statistics like data size @@ -63,5 +63,4 @@ public interface OutputContext extends TaskContext { * @param eventHandler should handle the events after the call. */ void trapEvents(EventHandler eventHandler); - } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputFrameworkInterface.java b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputFrameworkInterface.java index 617deee914..1bf8a0816b 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputFrameworkInterface.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputFrameworkInterface.java @@ -27,16 +27,15 @@ /** * Represents the Tez framework part of an {@link org.apache.tez.runtime.api.Output}. *

- * + *

* This interface has methods which are used by the Tez framework to control the Output. *

- * + *

* During initialization, Outputs must specify an initial memory requirement via * {@link OutputContext}.requestInitialMemory *

- * + *

* Users are expected to derive from {@link AbstractLogicalOutput} - * */ @Public public interface OutputFrameworkInterface { @@ -45,8 +44,7 @@ public interface OutputFrameworkInterface { * Initializes the Output * * @return list of events that were generated during initialization - * @throws Exception - * if an error occurs + * @throws Exception if an error occurs */ public List initialize() throws Exception; @@ -54,8 +52,7 @@ public interface OutputFrameworkInterface { * Handles user and system generated {@link Event}s, which typically carry * information such as a downstream vertex being ready to consume input. * - * @param outputEvents - * the list of {@link Event}s + * @param outputEvents the list of {@link Event}s */ public void handleEvents(List outputEvents); @@ -63,8 +60,7 @@ public interface OutputFrameworkInterface { * Closes the Output * * @return list of events that were generated during close - * @throws Exception - * if an error occurs + * @throws Exception if an error occurs */ public List close() throws Exception; } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatistics.java b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatistics.java index 2f18a03634..09cf14c4df 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatistics.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatistics.java @@ -1,24 +1,23 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.runtime.api; - import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.tez.dag.api.Vertex; @@ -31,20 +30,19 @@ @Public @Evolving public interface OutputStatistics { - + /** * Returns the data size associated with this logical output *
It is the size of the data written to this output by the vertex. * @return Data size in bytes */ - public long getDataSize(); - - /** + public long getDataSize(); + + /** * Get the numbers of items processed. These could be key-value pairs, table * records etc. - * + * * @return Number of items processed - */ - public long getItemsProcessed(); - + */ + public long getItemsProcessed(); } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatisticsReporter.java b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatisticsReporter.java index 1931e5cae3..2b8b977bd2 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatisticsReporter.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/OutputStatisticsReporter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -37,10 +37,9 @@ public interface OutputStatisticsReporter { /** * Report the number of items processed. These could be key-value pairs, table * records etc. - * + * * @param items * number of items */ public void reportItemsProcessed(long items); - } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/Processor.java b/tez-api/src/main/java/org/apache/tez/runtime/api/Processor.java index 32d786e3d4..461073d01c 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/Processor.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/Processor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,7 +24,7 @@ * {@link Processor} represents the Tez entity responsible for * consuming {@link Input} and producing {@link Output}. It defines the * application logic that processes and transforms the data. - * + * * Users are expected to derive from {@link AbstractLogicalIOProcessor} */ @Public diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/ProcessorContext.java b/tez-api/src/main/java/org/apache/tez/runtime/api/ProcessorContext.java index 3782a8d8d7..6d99a32382 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/ProcessorContext.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/ProcessorContext.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -43,7 +43,7 @@ static float preProcessProgress(float progress) { /** * Set the overall progress of this Task Attempt. - * This automatically results in invocation of {@link ProcessorContext#notifyProgress()} + * This automatically results in invocation of {@link ProcessorContext#notifyProgress()} * and so invoking that separately is not required. * @param progress Progress in the range from [0.0 - 1.0f] */ @@ -68,15 +68,15 @@ default void setProgress(float progress) { /** * Blocking call which returns when any of the specified Inputs is ready for * consumption. - * + * * There can be multiple parallel invocations of this function - where each * invocation blocks on the Inputs that it specifies. - * + * * If multiple Inputs are ready, any one of them may be returned by this * method - including an Input which may have been returned in a previous * call. If invoking this method multiple times, it's recommended to remove * previously completed Inputs from the invocation list. - * + * * @param inputs * the list of Inputs to monitor * @return the Input which is ready for consumption @@ -110,10 +110,10 @@ default void setProgress(float progress) { /** * Blocking call which returns only after all of the specified Inputs are * ready for consumption. - * + * * There can be multiple parallel invocations of this function - where each * invocation blocks on the Inputs that it specifies. - * + * * @param inputs * the list of Inputs to monitor * @throws InterruptedException diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/ProcessorFrameworkInterface.java b/tez-api/src/main/java/org/apache/tez/runtime/api/ProcessorFrameworkInterface.java index 89d4e3cbf6..5ef1aeac3d 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/ProcessorFrameworkInterface.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/ProcessorFrameworkInterface.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,7 +23,6 @@ import java.util.List; - /** * Represents the Tez framework part of an {@link org.apache.tez.runtime.api.Processor}. *

diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/ProgressFailedException.java b/tez-api/src/main/java/org/apache/tez/runtime/api/ProgressFailedException.java index 07995cc796..edbc165f23 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/ProgressFailedException.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/ProgressFailedException.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -42,5 +42,4 @@ public ProgressFailedException(Throwable cause) { public ProgressFailedException(String message, Throwable cause) { super(message, cause); } - } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/Reader.java b/tez-api/src/main/java/org/apache/tez/runtime/api/Reader.java index 9fc5094694..80d6a969e1 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/Reader.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/Reader.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/TaskAttemptIdentifier.java b/tez-api/src/main/java/org/apache/tez/runtime/api/TaskAttemptIdentifier.java index 101fa91d36..6ab81c6c11 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/TaskAttemptIdentifier.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/TaskAttemptIdentifier.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,8 +19,8 @@ package org.apache.tez.runtime.api; public interface TaskAttemptIdentifier { - + public int getIdentifier(); - + public TaskIdentifier getTaskIdentifier(); } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/TaskContext.java b/tez-api/src/main/java/org/apache/tez/runtime/api/TaskContext.java index 1ba1a90e3e..2b7c353ddb 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/TaskContext.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/TaskContext.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -80,7 +80,7 @@ public interface TaskContext { * @return Vertex Name */ public String getTaskVertexName(); - + /** * Get the index of this task's vertex in the set of vertices in the DAG. This * is consistent and valid across all tasks/vertices in the same DAG. @@ -119,18 +119,18 @@ public interface TaskContext { /** * Returns an identifier which is unique to the specific Input, Processor or * Output - * + * * @return a unique identifier */ public String getUniqueIdentifier(); - + /** * Returns a shared {@link ObjectRegistry} to hold user objects in memory * between tasks. * @return {@link ObjectRegistry} */ public ObjectRegistry getObjectRegistry(); - + /** * Notifies the framework that progress is being made by this component. * This is used to identify hung components that are not making progress. @@ -158,7 +158,6 @@ public interface TaskContext { @Deprecated public void fatalError(@Nullable Throwable exception, @Nullable String message); - /** * Report an error to the framework. This will cause the entire task to be terminated. * @@ -198,34 +197,34 @@ public interface TaskContext { */ @Nullable public ByteBuffer getServiceProviderMetaData(String serviceName); - + /** * Request a specific amount of memory during initialization * (initialize(..*Context)) The requester is notified of allocation via the * provided callback handler. - * + * * Currently, (post TEZ-668) the caller will be informed about the available * memory after initialization (I/P/O initialize(...)), and before the * start/run invocation. There will be no other invocations on the callback. - * + * * This method can be called only once by any component. Calling it multiple * times from within the same component will result in an error. - * + * * Each Input / Output must request memory. For Inputs / Outputs which do not * have a specific ask, a null callback handler can be specified with a * request size of 0. - * + * * @param size * request size in bytes. * @param callbackHandler * the callback handler to be invoked once memory is assigned */ public void requestInitialMemory(long size, MemoryUpdateCallback callbackHandler); - + /** * Gets the total memory available to all components of the running task. This * values will always be constant, and does not factor in any allocations. - * + * * @return the total available memory for all components of the task */ public long getTotalMemoryAvailableToTask(); diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/TaskIdentifier.java b/tez-api/src/main/java/org/apache/tez/runtime/api/TaskIdentifier.java index 8ef066bb7f..b4557b90e9 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/TaskIdentifier.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/TaskIdentifier.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,8 +19,8 @@ package org.apache.tez.runtime.api; public interface TaskIdentifier { - + public int getIdentifier(); - + public VertexIdentifier getVertexIdentifier(); } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/VertexIdentifier.java b/tez-api/src/main/java/org/apache/tez/runtime/api/VertexIdentifier.java index 16e88ad4ec..72878fd5e8 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/VertexIdentifier.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/VertexIdentifier.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,8 +21,8 @@ public interface VertexIdentifier { public int getIdentifier(); - + public String getName(); - + public DagIdentifier getDagIdentifier(); } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/VertexStatistics.java b/tez-api/src/main/java/org/apache/tez/runtime/api/VertexStatistics.java index aa526f24b5..bad50f9a3e 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/VertexStatistics.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/VertexStatistics.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.runtime.api; @@ -38,7 +38,7 @@ public interface VertexStatistics { /** * Get statistics about an {@link Edge} input or external input of this * {@link Vertex}.
- * + * * @param inputName * Name of the input {@link Edge} or external input of this vertex * @return {@link InputStatistics} for the given input @@ -48,11 +48,10 @@ public interface VertexStatistics { /** * Get statistics about an {@link Edge} output or external output of this * {@link Vertex}.
- * + * * @param outputName * Name of the output {@link Edge} or external output of this vertex * @return {@link OutputStatistics} for the given output */ public OutputStatistics getOutputStatistics(String outputName); - } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/Writer.java b/tez-api/src/main/java/org/apache/tez/runtime/api/Writer.java index f418b708dc..4fe934c642 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/Writer.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/Writer.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java index 32089a99ac..9d674267c3 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,14 +29,14 @@ /** * A convenience class to specify multiple DataMovementEvents which share the * same payload. A contiguous range of srcIndices can be specified. - * + * * This event will NOT be seen by downstream Inputs - instead they will see * {@link DataMovementEvent}s which are generated based on the range specified * in this event. - * + * * This event should be used by an output which has the same payload for all of * the Physical Outputs that it generates. - * + * */ @Public public class CompositeDataMovementEvent extends Event { @@ -67,12 +67,12 @@ public static CompositeDataMovementEvent create(int srcIndexStart, int count, ByteBuffer userPayload) { return new CompositeDataMovementEvent(srcIndexStart, count, userPayload); } - + /** * Expand the {@link CompositeDataMovementEvent} into a routable * {@link DataMovementEvent} by providing the source output index and the * target input index. - * + * * @param sourceIndex * The index of the physical output represented by the * {@link DataMovementEvent} @@ -111,11 +111,11 @@ public int getVersion() { public Iterable getEvents() { return new Iterable() { - + @Override public Iterator iterator() { return new Iterator() { - + int currentPos = sourceIndexStart; @Override @@ -144,5 +144,4 @@ public void remove() { public CompositeRoutedDataMovementEvent expandRouted(CompositeEventRouteMetadata routeMeta) { return CompositeRoutedDataMovementEvent.create(routeMeta.getSource(), routeMeta.getTarget(), routeMeta.getCount(), version, userPayload); } - } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeRoutedDataMovementEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeRoutedDataMovementEvent.java index 6dbed71e7e..f50f695be0 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeRoutedDataMovementEvent.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeRoutedDataMovementEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -55,7 +55,6 @@ public final class CompositeRoutedDataMovementEvent extends Event { */ private int version; - @Private public static CompositeRoutedDataMovementEvent create(int sourceIndex, int targetIndex, diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/CustomProcessorEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/CustomProcessorEvent.java index 7d84a1d79d..1aa45a9a9e 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/CustomProcessorEvent.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/CustomProcessorEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/DataMovementEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/DataMovementEvent.java index 05c3d3fe4f..b2f51be3bb 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/DataMovementEvent.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/DataMovementEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -56,12 +56,11 @@ public final class DataMovementEvent extends Event { */ private int version; - @Private DataMovementEvent(int sourceIndex, - int targetIndex, - int version, - ByteBuffer userPayload) { + int targetIndex, + int version, + ByteBuffer userPayload) { this.userPayload = userPayload; this.sourceIndex = sourceIndex; this.version = version; @@ -82,7 +81,7 @@ public static DataMovementEvent create(int sourceIndex, ByteBuffer userPayload) { return new DataMovementEvent(sourceIndex, -1, -1, userPayload); } - + @Private /** * Constructor for Processor-generated User Events @@ -99,11 +98,11 @@ public static DataMovementEvent create(int sourceIndex, ByteBuffer userPayload) { return new DataMovementEvent(sourceIndex, targetIndex, version, userPayload); } - + /** * Make a routable copy of the {@link DataMovementEvent} by adding a target * input index - * + * * @param targetIndex * The index of the physical input to which this * {@link DataMovementEvent} should be routed diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputConfigureVertexTasksEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputConfigureVertexTasksEvent.java index 05857d286c..4241190f3a 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputConfigureVertexTasksEvent.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputConfigureVertexTasksEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -61,5 +61,4 @@ public VertexLocationHint getLocationHint() { public InputSpecUpdate getInputSpecUpdate() { return this.inputSpecUpdate; } - } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputDataInformationEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputDataInformationEvent.java index a62a34154f..b2e50a7c38 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputDataInformationEvent.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputDataInformationEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -30,14 +30,14 @@ * Events used by {@link InputInitializer} implementations to provide the * user payload for individual tasks running as part of the Vertex for which an * Initial Input has been configured. - * + * * This event is used by InputInitialziers to configure tasks belonging to a * Vertex. The event may be processed by a @link {@link VertexManagerPlugin} * before being sent to tasks. - * + * * A {@link InputInitializer} may send Events with or without a * serialized user payload. - * + * * Events, after being processed by a {@link VertexManagerPlugin}, must * contain the payload in a serialized form. */ @@ -45,19 +45,17 @@ @Public public final class InputDataInformationEvent extends Event { - private final int sourceIndex; private int targetIndex; // TODO Likely to be multiple at a later point. private final ByteBuffer userPayload; private final Object userPayloadObject; - private InputDataInformationEvent(int srcIndex, ByteBuffer userPayload) { this.sourceIndex = srcIndex; this.userPayload = userPayload; this.userPayloadObject = null; } - + private InputDataInformationEvent(int srcIndex, Object userPayloadDeserialized, Object sigChanged) { this.sourceIndex = srcIndex; this.userPayloadObject = userPayloadDeserialized; @@ -90,11 +88,11 @@ public int getTargetIndex() { public void setTargetIndex(int target) { this.targetIndex = target; } - + public ByteBuffer getUserPayload() { return userPayload == null ? null : userPayload.asReadOnlyBuffer(); } - + public Object getDeserializedUserPayload() { return this.userPayloadObject; } @@ -104,5 +102,5 @@ public String toString() { return "InputDataInformationEvent [sourceIndex=" + sourceIndex + ", targetIndex=" + targetIndex + ", serializedUserPayloadExists=" + (userPayload != null) + ", deserializedUserPayloadExists=" + (userPayloadObject != null) + "]"; - } + } } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputFailedEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputFailedEvent.java index 9d8363a785..3d0057e72a 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputFailedEvent.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputFailedEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -28,7 +28,7 @@ * Users are not expected to send this event. */ @Private -public class InputFailedEvent extends Event{ +public class InputFailedEvent extends Event { /** * Index(i) of the i-th (physical) Input or Output that is meant to receive @@ -40,11 +40,11 @@ public class InputFailedEvent extends Event{ * Version number to indicate what attempt generated this Event */ private int version; - + @Private // for Writable public InputFailedEvent() { } - + @Private private InputFailedEvent(int targetIndex, int version) { this.targetIndex = targetIndex; @@ -55,15 +55,15 @@ private InputFailedEvent(int targetIndex, int version) { public static InputFailedEvent create(int targetIndex, int version) { return new InputFailedEvent(targetIndex, version); } - + /** * Create a copy of the {@link InputFailedEvent} by adding a target input * index The index of the physical input to which this event should be routed - * + * * @param targetIndex * The index of the physical input to which this * {@link InputFailedEvent} should be routed - * + * * @return copy of the {@link InputFailedEvent} with the target input index * added */ diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputInitializerEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputInitializerEvent.java index 21ad130c88..4e7d3a2148 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputInitializerEvent.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputInitializerEvent.java @@ -33,7 +33,7 @@ * An event that is routed to the specified {@link InputInitializer}. * This can be used to send information/metadata to the {@link InputInitializer} *

- * + *

* These events are routed to the InputInitializer, only after the task which generated the event * succeeds. Also, the events will only be sent once per task - irrespective of how many attempts * were run, or succeeded. An example of this is when a task is retried because the node on which it @@ -105,6 +105,7 @@ public void setSourceVertexName(String srcVertexName) { /** * Returns the name of the vertex which generated the event. This will only be populated after * the event has been routed by the AM. + * * @return the name of the source vertex */ public String getSourceVertexName() { @@ -116,4 +117,4 @@ public String toString() { return "[TargetVertexName=" + targetVertexName + ", TargetInputName=" + targetInputName + ", SourceVertexName=" + sourceVertexName + " Payload=" + eventPayload + "]"; } -} \ No newline at end of file +} diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputReadErrorEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputReadErrorEvent.java index 6c9f1fbe6b..2a263293db 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputReadErrorEvent.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputReadErrorEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -70,7 +70,7 @@ public final class InputReadErrorEvent extends Event { private final String destinationLocalhostName; private InputReadErrorEvent(final String diagnostics, final int index, final int version, - final int numFailures, boolean isLocalFetch, boolean isDiskErrorAtSource, String destinationLocalhostName) { + final int numFailures, boolean isLocalFetch, boolean isDiskErrorAtSource, String destinationLocalhostName) { super(); this.diagnostics = diagnostics; this.index = index; @@ -94,7 +94,7 @@ public static InputReadErrorEvent create(String diagnostics, int index, int vers } public static InputReadErrorEvent create(String diagnostics, int index, int version, boolean isLocalFetch, - boolean isDiskErrorAtSource, String destinationLocalhostName) { + boolean isDiskErrorAtSource, String destinationLocalhostName) { return create(diagnostics, index, version, 1, isLocalFetch, isDiskErrorAtSource, destinationLocalhostName); } @@ -102,7 +102,7 @@ public static InputReadErrorEvent create(String diagnostics, int index, int vers * Create an InputReadErrorEvent. */ public static InputReadErrorEvent create(final String diagnostics, final int index, final int version, - final int numFailures, boolean isLocalFetch, boolean isDiskErrorAtSource, String destinationLocalhostName) { + final int numFailures, boolean isLocalFetch, boolean isDiskErrorAtSource, String destinationLocalhostName) { return new InputReadErrorEvent(diagnostics, index, version, numFailures, isLocalFetch, isDiskErrorAtSource, destinationLocalhostName); } @@ -134,7 +134,7 @@ public boolean isDiskErrorAtSource() { return isDiskErrorAtSource; } - public String getDestinationLocalhostName(){ + public String getDestinationLocalhostName() { return destinationLocalhostName; } diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputUpdatePayloadEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputUpdatePayloadEvent.java index 8c336354d9..a447fbb49c 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputUpdatePayloadEvent.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputUpdatePayloadEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/VertexManagerEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/VertexManagerEvent.java index 7a450c6cdf..03038b22cc 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/VertexManagerEvent.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/VertexManagerEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -41,9 +41,9 @@ public class VertexManagerEvent extends Event { * Vertex to which the event should be sent */ private final String targetVertexName; - + private TaskAttemptIdentifier producerAttempt; - + /** * User payload to be sent */ @@ -74,11 +74,11 @@ public static VertexManagerEvent create(String vertexName, ByteBuffer userPayloa public String getTargetVertexName() { return targetVertexName; } - + public ByteBuffer getUserPayload() { return userPayload == null ? null : userPayload.asReadOnlyBuffer(); } - + /** * Get metadata about the task attempt that produced the event. * This method will provide a valid return value only when invoked in the @@ -88,7 +88,7 @@ public ByteBuffer getUserPayload() { public TaskAttemptIdentifier getProducerAttemptIdentifier() { return producerAttempt; } - + @Private public void setProducerAttemptIdentifier(TaskAttemptIdentifier producerAttempt) { this.producerAttempt = producerAttempt; diff --git a/tez-api/src/main/java/org/apache/tez/runtime/library/common/shuffle/api/ShuffleHandlerError.java b/tez-api/src/main/java/org/apache/tez/runtime/library/common/shuffle/api/ShuffleHandlerError.java index 09137de673..ae85f798fc 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/library/common/shuffle/api/ShuffleHandlerError.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/library/common/shuffle/api/ShuffleHandlerError.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.runtime.library.common.shuffle.api; /** diff --git a/tez-api/src/main/java/org/apache/tez/runtime/library/common/shuffle/api/package-info.java b/tez-api/src/main/java/org/apache/tez/runtime/library/common/shuffle/api/package-info.java index 9ad8e61d50..a2cdde3858 100644 --- a/tez-api/src/main/java/org/apache/tez/runtime/library/common/shuffle/api/package-info.java +++ b/tez-api/src/main/java/org/apache/tez/runtime/library/common/shuffle/api/package-info.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,4 +19,4 @@ @Private package org.apache.tez.runtime.library.common.shuffle.api; -import org.apache.hadoop.classification.InterfaceAudience.Private; \ No newline at end of file +import org.apache.hadoop.classification.InterfaceAudience.Private; diff --git a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLaunchRequest.java b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLaunchRequest.java index 63179c587c..f91ba9ab47 100644 --- a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLaunchRequest.java +++ b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLaunchRequest.java @@ -46,11 +46,12 @@ public ContainerLaunchRequest(NodeId nodeId, this.taskCommName = taskCommName; } - // TODO Post TEZ-2003. TEZ-2625. ContainerLaunchContext needs to be built here instead of being passed in. // Basic specifications need to be provided here + /** * The {@link ContainerLauncherContext} for the container being launched + * * @return the container launch context for the launch request */ public ContainerLaunchContext getContainerLaunchContext() { @@ -60,6 +61,7 @@ public ContainerLaunchContext getContainerLaunchContext() { /** * Get the name of the task communicator which will be used to communicate * with the task that will run in this container. + * * @return the task communicator to be used for this request */ public String getTaskCommunicatorName() { @@ -68,6 +70,7 @@ public String getTaskCommunicatorName() { /** * Get the name of the scheduler which allocated this container. + * * @return the scheduler name which provided the container */ public String getSchedulerName() { diff --git a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLauncher.java b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLauncher.java index 8792fd73e0..b53a43b1e3 100644 --- a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLauncher.java +++ b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLauncher.java @@ -79,7 +79,7 @@ public final ContainerLauncherContext getContext() { * * @param launchRequest the actual launch request * @throws ServicePluginException when the service runs into a fatal error which it cannot handle. - * This will cause the app to shutdown. + * This will cause the app to shutdown. */ public abstract void launchContainer(ContainerLaunchRequest launchRequest) throws ServicePluginException; @@ -89,7 +89,7 @@ public abstract void launchContainer(ContainerLaunchRequest launchRequest) throw * * @param stopRequest the actual stop request * @throws ServicePluginException when the service runs into a fatal error which it cannot handle. - * This will cause the app to shutdown. + * This will cause the app to shutdown. */ public abstract void stopContainer(ContainerStopRequest stopRequest) throws ServicePluginException; } diff --git a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLauncherContext.java b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLauncherContext.java index 16d54f05b7..328992e43d 100644 --- a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLauncherContext.java +++ b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLauncherContext.java @@ -14,7 +14,6 @@ package org.apache.tez.serviceplugins.api; - import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -98,8 +97,6 @@ void containerCompleted(ContainerId containerId, int exitStatus, String diagnost * * @param taskCommName the name of the task communicator * @return meta info for the requested task communicator - * */ Object getTaskCommunicatorMetaInfo(String taskCommName); - } diff --git a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLauncherOperationBase.java b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLauncherOperationBase.java index 98806fabbf..2bc6c75a53 100644 --- a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLauncherOperationBase.java +++ b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ContainerLauncherOperationBase.java @@ -28,7 +28,6 @@ public class ContainerLauncherOperationBase { // - Get rid of YARN constructs. // - ContainerToken may not always be required - private final NodeId nodeId; private final ContainerId containerId; private final Token containerToken; @@ -43,6 +42,7 @@ public ContainerLauncherOperationBase(NodeId nodeId, /** * Get the node on which this container is to be launched + * * @return the node id for the container */ public NodeId getNodeId() { @@ -51,6 +51,7 @@ public NodeId getNodeId() { /** * Get the containerId for the container + * * @return the container id for the container opeartion */ public ContainerId getContainerId() { @@ -59,6 +60,7 @@ public ContainerId getContainerId() { /** * Get the security token for the container. Primarily for YARN + * * @return the token for the container launch. */ public Token getContainerToken() { diff --git a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/DagInfo.java b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/DagInfo.java index b05fa8bbb7..fbd83b1167 100644 --- a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/DagInfo.java +++ b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/DagInfo.java @@ -23,18 +23,21 @@ public interface DagInfo { /** * The index of the current dag + * * @return a numerical identifier for the DAG. This is unique within the currently running application. */ int getIndex(); /** * Get the name of the dag + * * @return the name of the dag */ String getName(); /** * Get credentials for the dag. + * * @return the credentials for the dag. */ Credentials getCredentials(); diff --git a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginContextBase.java b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginContextBase.java index 90a51b20dc..a95d92db79 100644 --- a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginContextBase.java +++ b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginContextBase.java @@ -33,6 +33,7 @@ public interface ServicePluginContextBase { /** * Get information on the currently executing dag + * * @return info on the currently running dag, or null if no dag is executing */ @Nullable @@ -42,8 +43,8 @@ public interface ServicePluginContextBase { * Report an error from the service. This results in the specific DAG being killed. * * @param servicePluginError the error category - * @param message A diagnostic message associated with this error - * @param dagInfo the affected dag + * @param message A diagnostic message associated with this error + * @param dagInfo the affected dag */ void reportError(@Nonnull ServicePluginError servicePluginError, String message, DagInfo dagInfo); } diff --git a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginError.java b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginError.java index 932c0faa3e..35a51caefe 100644 --- a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginError.java +++ b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginError.java @@ -43,6 +43,4 @@ enum ErrorType { * @return the type of the error */ ErrorType getErrorType(); - - } diff --git a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginErrorDefaults.java b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginErrorDefaults.java index 83a85b5cf2..9b95f27a23 100644 --- a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginErrorDefaults.java +++ b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginErrorDefaults.java @@ -30,10 +30,9 @@ /** * A default set of errors from ServicePlugins - * + *

* Errors are marked as fatal or non-fatal for the Application. * Fatal errors cause the AM to go down. - * */ @InterfaceAudience.Public public enum ServicePluginErrorDefaults implements ServicePluginError { @@ -43,7 +42,8 @@ public enum ServicePluginErrorDefaults implements ServicePluginError { */ SERVICE_UNAVAILABLE(ErrorType.TEMPORARY), - /** Indicates that the service is in an inconsistent state. + /** + * Indicates that the service is in an inconsistent state. * This is a fatal error. */ INCONSISTENT_STATE(ErrorType.PERMANENT), diff --git a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginsDescriptor.java b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginsDescriptor.java index c942a3ad3a..decc479d97 100644 --- a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginsDescriptor.java +++ b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/ServicePluginsDescriptor.java @@ -57,9 +57,9 @@ private ServicePluginsDescriptor(boolean enableContainers, boolean enableUber, * Create a service plugin descriptor with the provided plugins. Regular containers will also be enabled * when using this method. * - * @param taskSchedulerDescriptor the task scheduler plugin descriptors + * @param taskSchedulerDescriptor the task scheduler plugin descriptors * @param containerLauncherDescriptors the container launcher plugin descriptors - * @param taskCommunicatorDescriptors the task communicator plugin descriptors + * @param taskCommunicatorDescriptors the task communicator plugin descriptors * @return a {@link ServicePluginsDescriptor} instance */ public static ServicePluginsDescriptor create(TaskSchedulerDescriptor[] taskSchedulerDescriptor, @@ -72,17 +72,17 @@ public static ServicePluginsDescriptor create(TaskSchedulerDescriptor[] taskSche /** * Create a service plugin descriptor with the provided plugins. Also allows specification of whether * in-AM execution is enabled. Container execution is enabled by default. - * + *

* Note on Uber mode: This is NOT fully supported at the moment. Tasks will be launched within the * AM process itself, controlled by {@link TezConfiguration#TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS}. * The AM will need to be sized correctly for the tasks. Memory allocation to the running task * cannot be controlled yet, and is the full AM heap for each task. * TODO: TEZ-2722 * - * @param enableUber whether to enable execution in the AM or not - * @param taskSchedulerDescriptor the task scheduler plugin descriptors + * @param enableUber whether to enable execution in the AM or not + * @param taskSchedulerDescriptor the task scheduler plugin descriptors * @param containerLauncherDescriptors the container launcher plugin descriptors - * @param taskCommunicatorDescriptors the task communicator plugin descriptors + * @param taskCommunicatorDescriptors the task communicator plugin descriptors * @return a {@link ServicePluginsDescriptor} instance */ public static ServicePluginsDescriptor create(boolean enableUber, @@ -96,18 +96,18 @@ public static ServicePluginsDescriptor create(boolean enableUber, /** * Create a service plugin descriptor with the provided plugins. Also allows specification of whether * container execution and in-AM execution will be enabled. - * + *

* Note on Uber mode: This is NOT fully supported at the moment. Tasks will be launched within the * AM process itself, controlled by {@link TezConfiguration#TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS}. * The AM will need to be sized correctly for the tasks. Memory allocation to the running task * cannot be controlled yet, and is the full AM heap for each task. * TODO: TEZ-2722 * - * @param enableContainers whether to enable execution in containers - * @param enableUber whether to enable execution in the AM or not - * @param taskSchedulerDescriptor the task scheduler plugin descriptors + * @param enableContainers whether to enable execution in containers + * @param enableUber whether to enable execution in the AM or not + * @param taskSchedulerDescriptor the task scheduler plugin descriptors * @param containerLauncherDescriptors the container launcher plugin descriptors - * @param taskCommunicatorDescriptors the task communicator plugin descriptors + * @param taskCommunicatorDescriptors the task communicator plugin descriptors * @return a {@link ServicePluginsDescriptor} instance */ public static ServicePluginsDescriptor create(boolean enableContainers, boolean enableUber, @@ -121,7 +121,7 @@ public static ServicePluginsDescriptor create(boolean enableContainers, boolean /** * Create a service plugin descriptor which may have in-AM execution of tasks enabled. Container * execution is enabled by default - * + *

* Note on Uber mode: This is NOT fully supported at the moment. Tasks will be launched within the * AM process itself, controlled by {@link TezConfiguration#TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS}. * The AM will need to be sized correctly for the tasks. Memory allocation to the running task @@ -135,7 +135,6 @@ public static ServicePluginsDescriptor create(boolean enableUber) { return new ServicePluginsDescriptor(true, enableUber, null, null, null); } - @InterfaceAudience.Private public boolean areContainersEnabled() { return enableContainers; @@ -171,4 +170,4 @@ public String toString() { ", taskCommunicatorDescriptors=" + Arrays.toString(taskCommunicatorDescriptors) + '}'; } -} \ No newline at end of file +} diff --git a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskCommunicatorDescriptor.java b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskCommunicatorDescriptor.java index 57ac385fca..da8dbc4515 100644 --- a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskCommunicatorDescriptor.java +++ b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskCommunicatorDescriptor.java @@ -22,7 +22,6 @@ @InterfaceStability.Unstable public class TaskCommunicatorDescriptor extends NamedEntityDescriptor { - private TaskCommunicatorDescriptor(String taskCommName, String taskCommClassname) { super(taskCommName, taskCommClassname); } diff --git a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskScheduler.java b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskScheduler.java index b28a684de5..2a35abb7f3 100644 --- a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskScheduler.java +++ b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskScheduler.java @@ -106,7 +106,7 @@ public final TaskSchedulerContext getContext() { * * @return the resources available at the time of invocation * @throws ServicePluginException when the service runs into a fatal error which it cannot handle. - * This will cause the app to shutdown. + * This will cause the app to shutdown. */ public abstract Resource getAvailableResources() throws ServicePluginException; @@ -115,7 +115,7 @@ public final TaskSchedulerContext getContext() { * * @return the total available resources from the source * @throws ServicePluginException when the service runs into a fatal error which it cannot handle. - * This will cause the app to shutdown. + * This will cause the app to shutdown. */ public abstract Resource getTotalResources() throws ServicePluginException; @@ -124,7 +124,7 @@ public final TaskSchedulerContext getContext() { * * @return the number of nodes * @throws ServicePluginException when the service runs into a fatal error which it cannot handle. - * This will cause the app to shutdown. + * This will cause the app to shutdown. */ public abstract int getClusterNodeCount() throws ServicePluginException; @@ -134,7 +134,7 @@ public final TaskSchedulerContext getContext() { * * @param nodeId te nodeId to be blacklisted * @throws ServicePluginException when the service runs into a fatal error which it cannot handle. - * This will cause the app to shutdown. + * This will cause the app to shutdown. */ public abstract void blacklistNode(NodeId nodeId) throws ServicePluginException; @@ -144,7 +144,7 @@ public final TaskSchedulerContext getContext() { * * @param nodeId the nodeId to be unblacklisted * @throws ServicePluginException when the service runs into a fatal error which it cannot handle. - * This will cause the app to shutdown. + * This will cause the app to shutdown. */ public abstract void unblacklistNode(NodeId nodeId) throws ServicePluginException; @@ -165,7 +165,7 @@ public final TaskSchedulerContext getContext() { * via the {@link TaskSchedulerContext#taskAllocated(Object, Object, * Container)} method when a task is assigned to a resource * @throws ServicePluginException when the service runs into a fatal error which it cannot handle. - * This will cause the app to shutdown. + * This will cause the app to shutdown. */ public abstract void allocateTask(Object task, Resource capability, String[] hosts, String[] racks, Priority priority, @@ -188,7 +188,7 @@ public abstract void allocateTask(Object task, Resource capability, * via the {@link TaskSchedulerContext#taskAllocated(Object, Object, * Container)} method when a task is assigned to a resource * @throws ServicePluginException when the service runs into a fatal error which it cannot handle. - * This will cause the app to shutdown. + * This will cause the app to shutdown. */ public abstract void allocateTask(Object task, Resource capability, ContainerId containerId, Priority priority, @@ -220,8 +220,8 @@ public void taskStateUpdated(Object task, SchedulerTaskState state) throws * @param diagnostics additional diagnostics information which may be relevant * @return true if the task was associated with a container, false if the task was not associated * @throws ServicePluginException when the service runs into a fatal error which it cannot handle. - * This will cause the app to shutdown. - * with a container + * This will cause the app to shutdown. + * with a container */ public abstract boolean deallocateTask(Object task, boolean taskSucceeded, TaskAttemptEndReason endReason, @@ -233,15 +233,16 @@ public abstract boolean deallocateTask(Object task, boolean taskSucceeded, * @param containerId the containerId to de-allocate * @return the task which was previously associated with this container, null otherwise * @throws ServicePluginException when the service runs into a fatal error which it cannot handle. - * This will cause the app to shutdown. + * This will cause the app to shutdown. */ public abstract Object deallocateContainer(ContainerId containerId) throws ServicePluginException; /** * Inform the scheduler that it should unregister. This is primarily valid for schedulers which * require registration (YARN a.t.m) + * * @throws ServicePluginException when the service runs into a fatal error which it cannot handle. - * This will cause the app to shutdown. + * This will cause the app to shutdown. */ public abstract void setShouldUnregister() throws ServicePluginException; @@ -250,7 +251,7 @@ public abstract boolean deallocateTask(Object task, boolean taskSucceeded, * * @return true if the scheduler has unregistered. False otherwise. * @throws ServicePluginException when the service runs into a fatal error which it cannot handle. - * This will cause the app to shutdown. + * This will cause the app to shutdown. */ public abstract boolean hasUnregistered() throws ServicePluginException; @@ -258,9 +259,9 @@ public abstract boolean deallocateTask(Object task, boolean taskSucceeded, * Indicates to the scheduler that the currently running dag has completed. * This can be used to reset dag specific statistics, potentially release resources and prepare * for a new DAG. + * * @throws ServicePluginException when the service runs into a fatal error which it cannot handle. - * This will cause the app to shutdown. + * This will cause the app to shutdown. */ public abstract void dagComplete() throws ServicePluginException; - } diff --git a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskSchedulerContext.java b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskSchedulerContext.java index e188231e14..46167b84a8 100644 --- a/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskSchedulerContext.java +++ b/tez-api/src/main/java/org/apache/tez/serviceplugins/api/TaskSchedulerContext.java @@ -71,7 +71,6 @@ enum AMState { // - setApplicationRegistrationData may not be relevant to non YARN clusters // - getAppFinalStatus may not be relevant to non YARN clusters - /** * Indicate to the framework that a container is being assigned to a task. * @@ -84,7 +83,6 @@ void taskAllocated(Object task, Object appCookie, Container container); - /** * Indicate to the framework that a container has completed. This is typically used by sources * which have @@ -105,7 +103,6 @@ void containerCompleted(Object taskLastAllocated, */ void containerBeingReleased(ContainerId containerId); - /** * Provide an update to the framework about the status of nodes available to this report * @@ -152,7 +149,6 @@ void setApplicationRegistrationData( */ AppFinalStatus getFinalAppStatus(); - // Getters /** diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java index 4f33348f89..7856e9229a 100644 --- a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java +++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -117,7 +117,7 @@ class TezClientForTest extends TezClient { Long prewarmTimeoutMs; public TezClientForTest(String name, TezConfiguration tezConf, - @Nullable Map localResources, @Nullable Credentials credentials) { + @Nullable Map localResources, @Nullable Credentials credentials) { super(name, tezConf, localResources, credentials); } @@ -146,7 +146,7 @@ protected TezYarnClientForTest(YarnClient yarnClient, TezClientForTest client) { @Override protected DAGClientAMProtocolBlockingPB waitForProxy(long clientTimeout, Configuration conf, - ApplicationId sessionAppId, UserGroupInformation ugi) throws TezException, IOException { + ApplicationId sessionAppId, UserGroupInformation ugi) throws TezException, IOException { if (!client.callRealGetSessionAMProxy) { return client.sessionAmProxy; } @@ -155,7 +155,7 @@ protected DAGClientAMProtocolBlockingPB waitForProxy(long clientTimeout, Configu @Override protected DAGClientAMProtocolBlockingPB getProxy(Configuration conf, ApplicationId sessionAppId, - UserGroupInformation ugi) throws TezException, IOException { + UserGroupInformation ugi) throws TezException, IOException { if (!client.callRealGetSessionAMProxy) { return client.sessionAmProxy; } @@ -171,7 +171,7 @@ TezClientForTest configureAndCreateTezClient(TezConfiguration conf) throws YarnE IOException { return configureAndCreateTezClient(new HashMap(), true, conf); } - + TezClientForTest configureAndCreateTezClient(Map lrs, boolean isSession, TezConfiguration conf) throws YarnException, IOException, ServiceException { if (conf == null) { @@ -200,45 +200,45 @@ TezClientForTest configureAndCreateTezClient(Map lrs, boo return client; } - - @Test (timeout = 5000) + + @Test(timeout = 5000) public void testTezclientApp() throws Exception { testTezClient(false, true); } - - @Test (timeout = 5000) + + @Test(timeout = 5000) public void testTezclientSession() throws Exception { testTezClient(true, true); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testTezClientSessionLargeDAGPlan() throws Exception { // request size is within threshold of being serialized - _testTezClientSessionLargeDAGPlan(10*1024*1024, 10, 10, false); + _testTezClientSessionLargeDAGPlan(10 * 1024 * 1024, 10, 10, false); // DAGPlan exceeds the threshold but is still less than max IPC size - _testTezClientSessionLargeDAGPlan(10*1024*1024, 6*1024*1024, 10, true); + _testTezClientSessionLargeDAGPlan(10 * 1024 * 1024, 6 * 1024 * 1024, 10, true); // DAGPlan exceeds max IPC size - _testTezClientSessionLargeDAGPlan(10*1024*1024, 15*1024*1024, 10, true); + _testTezClientSessionLargeDAGPlan(10 * 1024 * 1024, 15 * 1024 * 1024, 10, true); // amResources exceeds the threshold but is still less than max IPC size - _testTezClientSessionLargeDAGPlan(10*1024*1024, 10, 6*1024*1024, true); + _testTezClientSessionLargeDAGPlan(10 * 1024 * 1024, 10, 6 * 1024 * 1024, true); // amResources exceeds max IPC size - _testTezClientSessionLargeDAGPlan(10*1024*1024, 10, 15*1024*1024, true); + _testTezClientSessionLargeDAGPlan(10 * 1024 * 1024, 10, 15 * 1024 * 1024, true); // DAGPlan and amResources together exceed threshold but less than IPC size - _testTezClientSessionLargeDAGPlan(10*1024*1024, 3*1024*1024, 3*1024*1024, true); + _testTezClientSessionLargeDAGPlan(10 * 1024 * 1024, 3 * 1024 * 1024, 3 * 1024 * 1024, true); // DAGPlan and amResources all exceed max IPC size - _testTezClientSessionLargeDAGPlan(10*1024*1024, 15*1024*1024, 15*1024*1024, true); + _testTezClientSessionLargeDAGPlan(10 * 1024 * 1024, 15 * 1024 * 1024, 15 * 1024 * 1024, true); } private void _testTezClientSessionLargeDAGPlan(int maxIPCMsgSize, int payloadSize, int amResourceSize, - boolean shouldSerialize) throws Exception { + boolean shouldSerialize) throws Exception { TezConfiguration conf = new TezConfiguration(); conf.setInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, maxIPCMsgSize); - conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, "target/"+this.getClass().getName()); + conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, "target/" + this.getClass().getName()); TezClientForTest client = configureAndCreateTezClient(null, true, conf); Map localResourceMap = new HashMap<>(); byte[] bytes = new byte[amResourceSize]; - Arrays.fill(bytes, (byte)1); + Arrays.fill(bytes, (byte) 1); String lrName = new String(bytes); localResourceMap.put(lrName, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); @@ -274,7 +274,7 @@ private void _testTezClientSessionLargeDAGPlan(int maxIPCMsgSize, int payloadSiz } } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testGetClient() throws Exception { /* BEGIN first TezClient usage without calling stop() */ TezClientForTest client = testTezClient(true, false); @@ -283,18 +283,18 @@ public void testGetClient() throws Exception { /* BEGIN reuse of AM from new TezClient */ ArgumentCaptor captor = ArgumentCaptor.forClass(ApplicationSubmissionContext.class); when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState()) - .thenReturn(YarnApplicationState.RUNNING); + .thenReturn(YarnApplicationState.RUNNING); //Reuse existing appId from first TezClient ApplicationId existingAppId = client.mockAppId; TezClientForTest client2 = configureAndCreateTezClient(null, true, - client.amConfig.getTezConfiguration()); + client.amConfig.getTezConfiguration()); String mockLR1Name = "LR1"; Map lrDAG = Collections.singletonMap(mockLR1Name, LocalResource - .newInstance(URL.newInstance("file", "localhost", 0, "/test1"), LocalResourceType.FILE, - LocalResourceVisibility.PUBLIC, 1, 1)); + .newInstance(URL.newInstance("file", "localhost", 0, "/test1"), LocalResourceType.FILE, + LocalResourceVisibility.PUBLIC, 1, 1)); Vertex vertex = Vertex.create("Vertex", ProcessorDescriptor.create("P"), 1, - Resource.newInstance(1, 1)); + Resource.newInstance(1, 1)); DAG dag = DAG.create("DAG").addVertex(vertex).addTaskLocalFiles(lrDAG); //Bind TezClient to existing app and submit a dag @@ -312,22 +312,22 @@ public void testGetClient() throws Exception { // Validate stop from new TezClient as normal */ client2.stop(); verify(client2.sessionAmProxy, times(1)).shutdownSession(any(), - any()); + any()); verify(client2.mockYarnClient, times(1)).stop(); /* END reuse of AM from new TezClient */ } - + public TezClientForTest testTezClient(boolean isSession, boolean shouldStop) throws Exception { Map lrs = Maps.newHashMap(); String lrName1 = "LR1"; lrs.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); - + TezClientForTest client = configureAndCreateTezClient(lrs, isSession, null); ArgumentCaptor captor = ArgumentCaptor.forClass(ApplicationSubmissionContext.class); when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState()) - .thenReturn(YarnApplicationState.RUNNING); + .thenReturn(YarnApplicationState.RUNNING); client.start(); verify(client.mockYarnClient, times(1)).init(any()); verify(client.mockYarnClient, times(1)).start(); @@ -344,7 +344,7 @@ public TezClientForTest testTezClient(boolean isSession, boolean shouldStop) thr } else { verify(client.mockYarnClient, times(0)).submitApplication(captor.capture()); } - + String mockLR1Name = "LR1"; Map lrDAG = Collections.singletonMap(mockLR1Name, LocalResource .newInstance(URL.newInstance("file", "localhost", 0, "/test1"), LocalResourceType.FILE, @@ -354,10 +354,10 @@ public TezClientForTest testTezClient(boolean isSession, boolean shouldStop) thr DAG dag = DAG.create("DAG").addVertex(vertex).addTaskLocalFiles(lrDAG); if (!isSession) { when(client.sessionAmProxy.getAMStatus(any(), any())) - .thenReturn(GetAMStatusResponseProto.newBuilder().setStatus(TezAppMasterStatusProto.SHUTDOWN).build()); + .thenReturn(GetAMStatusResponseProto.newBuilder().setStatus(TezAppMasterStatusProto.SHUTDOWN).build()); } DAGClient dagClient = client.submitDAG(dag); - + assertTrue(dagClient.getExecutionContext().contains(client.mockAppId.toString())); assertEquals(dagClient.getSessionIdentifierString(), client.mockAppId.toString()); @@ -377,24 +377,24 @@ public TezClientForTest testTezClient(boolean isSession, boolean shouldStop) thr assertTrue(context.getAMContainerSpec().getLocalResources().containsKey( lrName1)); } - + // add resources String lrName2 = "LR2"; lrs.clear(); lrs.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test2"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); client.addAppMasterLocalFiles(lrs); - + ApplicationId appId2 = ApplicationId.newInstance(0, 2); when(client.mockYarnClient.createApplication().getNewApplicationResponse().getApplicationId()) .thenReturn(appId2); - + when(client.mockYarnClient.getApplicationReport(appId2).getYarnApplicationState()) - .thenReturn(YarnApplicationState.RUNNING); + .thenReturn(YarnApplicationState.RUNNING); dag = DAG.create("DAG").addVertex( Vertex.create("Vertex", ProcessorDescriptor.create("P"), 1, Resource.newInstance(1, 1))); dagClient = client.submitDAG(dag); - + if (isSession) { // same app master verify(client.mockYarnClient, times(1)).submitApplication(captor.capture()); @@ -426,32 +426,32 @@ public TezClientForTest testTezClient(boolean isSession, boolean shouldStop) thr lrName2)); } - if(shouldStop) { + if (shouldStop) { client.stop(); if (isSession) { verify(client.sessionAmProxy, times(1)).shutdownSession(any(), - any()); + any()); } verify(client.mockYarnClient, times(1)).stop(); } return client; } - @Test (timeout=5000) + @Test(timeout = 5000) public void testPreWarm() throws Exception { TezClientForTest client = configureAndCreateTezClient(); client.start(); when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState()) .thenReturn(YarnApplicationState.RUNNING); - + when( client.sessionAmProxy.getAMStatus(any(), any())) .thenReturn(GetAMStatusResponseProto.newBuilder().setStatus(TezAppMasterStatusProto.READY).build()); PreWarmVertex vertex = PreWarmVertex.create("PreWarm", 1, Resource.newInstance(1, 1)); client.preWarm(vertex); - + ArgumentCaptor captor1 = ArgumentCaptor.forClass(SubmitDAGRequestProto.class); verify(client.sessionAmProxy, times(1)).submitDAG(any(), captor1.capture()); SubmitDAGRequestProto proto = captor1.getValue(); @@ -461,8 +461,7 @@ public void testPreWarm() throws Exception { client.stop(); } - - @Test (timeout=5000) + @Test(timeout = 5000) public void testPreWarmCloseStuck() throws Exception { TezClientForTest client = configureAndCreateTezClient(); client.setPrewarmTimeoutMs(10L); // Don't wait too long. @@ -479,21 +478,20 @@ public void testPreWarmCloseStuck() throws Exception { client.stop(); } - private void setClientToReportStoppedDags(TezClientForTest client) throws Exception { when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState()) - .thenReturn(YarnApplicationState.FINISHED); + .thenReturn(YarnApplicationState.FINISHED); when(client.sessionAmProxy.getDAGStatus(isNull(), any())) - .thenReturn(GetDAGStatusResponseProto.newBuilder().setDagStatus(DAGStatusProto.newBuilder() - .addDiagnostics("Diagnostics_0").setState(DAGStatusStateProto.DAG_SUCCEEDED) - .setDAGProgress(ProgressProto.newBuilder() - .setFailedTaskCount(0).setKilledTaskCount(0).setRunningTaskCount(0) - .setSucceededTaskCount(1).setTotalTaskCount(1).build()).build()).build()); + .thenReturn(GetDAGStatusResponseProto.newBuilder().setDagStatus(DAGStatusProto.newBuilder() + .addDiagnostics("Diagnostics_0").setState(DAGStatusStateProto.DAG_SUCCEEDED) + .setDAGProgress(ProgressProto.newBuilder() + .setFailedTaskCount(0).setKilledTaskCount(0).setRunningTaskCount(0) + .setSucceededTaskCount(1).setTotalTaskCount(1).build()).build()).build()); } - @Test (timeout=30000) + @Test(timeout = 30000) public void testPreWarmWithTimeout() throws Exception { - long startTime = 0 , endTime = 0; + long startTime = 0, endTime = 0; TezClientForTest client = configureAndCreateTezClient(); final TezClientForTest spyClient = spy(client); doCallRealMethod().when(spyClient).start(); @@ -506,7 +504,7 @@ public void testPreWarmWithTimeout() throws Exception { .thenReturn(YarnApplicationState.RUNNING); when( spyClient.sessionAmProxy.getAMStatus(any(), - any())) + any())) .thenReturn( GetAMStatusResponseProto.newBuilder().setStatus( TezAppMasterStatusProto.INITIALIZING).build()); @@ -524,12 +522,11 @@ public void testPreWarmWithTimeout() throws Exception { verify(spyClient, times(0)).submitDAG(any()); Assert.assertTrue("Unexpected Exception message", te.getMessage().contains("Tez AM not ready")); - } when( spyClient.sessionAmProxy.getAMStatus(any(), - any())) + any())) .thenReturn( GetAMStatusResponseProto.newBuilder().setStatus( TezAppMasterStatusProto.READY).build()); @@ -546,28 +543,28 @@ public void testPreWarmWithTimeout() throws Exception { Thread amStateThread = new Thread() { @Override public void run() { - CountDownLatch latch = new CountDownLatch(1); - try { - when( - spyClient.sessionAmProxy.getAMStatus((RpcController) any(), - (GetAMStatusRequestProto) any())) - .thenReturn( - GetAMStatusResponseProto.newBuilder().setStatus( - TezAppMasterStatusProto.INITIALIZING).build()); - latch.await(1000, TimeUnit.MILLISECONDS); - when( - spyClient.sessionAmProxy.getAMStatus((RpcController) any(), - (GetAMStatusRequestProto) any())) - .thenReturn( - GetAMStatusResponseProto.newBuilder().setStatus( - TezAppMasterStatusProto.READY).build()); - } catch (InterruptedException e) { - e.printStackTrace(); - } catch (ServiceException e) { - e.printStackTrace(); - } + CountDownLatch latch = new CountDownLatch(1); + try { + when( + spyClient.sessionAmProxy.getAMStatus((RpcController) any(), + (GetAMStatusRequestProto) any())) + .thenReturn( + GetAMStatusResponseProto.newBuilder().setStatus( + TezAppMasterStatusProto.INITIALIZING).build()); + latch.await(1000, TimeUnit.MILLISECONDS); + when( + spyClient.sessionAmProxy.getAMStatus((RpcController) any(), + (GetAMStatusRequestProto) any())) + .thenReturn( + GetAMStatusResponseProto.newBuilder().setStatus( + TezAppMasterStatusProto.READY).build()); + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (ServiceException e) { + e.printStackTrace(); } - }; + } + }; amStateThread.start(); startTime = Time.monotonicNow(); spyClient.preWarm(vertex, timeout, TimeUnit.MILLISECONDS); @@ -580,19 +577,19 @@ public void run() { client.stop(); } - @Test (timeout = 10000) + @Test(timeout = 10000) public void testMultipleSubmissions() throws Exception { testMultipleSubmissionsJob(false); testMultipleSubmissionsJob(true); } - + public void testMultipleSubmissionsJob(boolean isSession) throws Exception { TezClientForTest client1 = configureAndCreateTezClient(new HashMap(), isSession, null); when(client1.mockYarnClient.getApplicationReport(client1.mockAppId).getYarnApplicationState()) - .thenReturn(YarnApplicationState.RUNNING); + .thenReturn(YarnApplicationState.RUNNING); client1.start(); - + String mockLR1Name = "LR1"; Map lrDAG = Collections.singletonMap(mockLR1Name, LocalResource .newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, @@ -607,20 +604,20 @@ public void testMultipleSubmissionsJob(boolean isSession) throws Exception { // the dag resource will be added to the vertex once client1.submitDAG(dag); - + TezClientForTest client2 = configureAndCreateTezClient(); when(client2.mockYarnClient.getApplicationReport(client2.mockAppId).getYarnApplicationState()) - .thenReturn(YarnApplicationState.RUNNING); + .thenReturn(YarnApplicationState.RUNNING); client2.start(); - + // verify resubmission of same dag to new client (simulates submission error resulting in the // creation of a new client and resubmission of the DAG) client2.submitDAG(dag); - + client1.stop(); client2.stop(); } - + @Test(timeout = 5000) public void testWaitTillReady_Interrupt() throws Exception { final TezClientForTest client = configureAndCreateTezClient(); @@ -637,7 +634,9 @@ public void run() { } catch (Exception e) { exceptionReference.set(e); } - }; + } + + ; }; thread.start(); thread.join(250); @@ -646,7 +645,7 @@ public void run() { Assert.assertThat(exceptionReference.get(), CoreMatchers.instanceOf(InterruptedException.class)); client.stop(); } - + @Test(timeout = 5000) public void testWaitTillReadyAppFailed() throws Exception { final TezClientForTest client = configureAndCreateTezClient(); @@ -664,7 +663,7 @@ public void testWaitTillReadyAppFailed() throws Exception { } client.stop(); } - + @Test(timeout = 5000) public void testWaitTillReadyAppFailedNoDiagnostics() throws Exception { final TezClientForTest client = configureAndCreateTezClient(); @@ -679,12 +678,12 @@ public void testWaitTillReadyAppFailedNoDiagnostics() throws Exception { } client.stop(); } - + @Test(timeout = 5000) public void testSubmitDAGAppFailed() throws Exception { final TezClientForTest client = configureAndCreateTezClient(); client.start(); - + client.callRealGetSessionAMProxy = true; String msg = "Application Test Failed"; when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState()) @@ -695,7 +694,7 @@ public void testSubmitDAGAppFailed() throws Exception { Vertex vertex = Vertex.create("Vertex", ProcessorDescriptor.create("P"), 1, Resource.newInstance(1, 1)); DAG dag = DAG.create("DAG").addVertex(vertex); - + try { client.submitDAG(dag); fail(); @@ -718,7 +717,7 @@ public void testTezClientCounterLimits() throws YarnException, IOException, Serv configureAndCreateTezClient(conf); TezCounters counters = new TezCounters(); - for (int i = 0 ; i < newCounterLimit ; i++) { + for (int i = 0; i < newCounterLimit; i++) { counters.findCounter("GroupName", "TestCounter" + i).setValue(i); } @@ -889,7 +888,7 @@ public void testClientResubmit() throws Exception { try { client.submitDAG(dag); Assert.fail("Expected TezUncheckedException here."); - } catch(TezUncheckedException ex) { + } catch (TezUncheckedException ex) { Assert.assertTrue(ex.getMessage().contains("Invalid/conflicting GC options found")); } } @@ -972,7 +971,7 @@ public void testAMHeartbeatFailOnGetAMProxy_AppFailed() throws Exception { client.start(); when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState()) - .thenReturn(YarnApplicationState.FAILED); + .thenReturn(YarnApplicationState.FAILED); Thread.sleep(2 * amHeartBeatTimeoutSecs * 1000); assertTrue(client.getAMKeepAliveService().isTerminated()); } @@ -989,7 +988,7 @@ public void testAMHeartbeatFailOnGetAMStatus() throws Exception { when(client.sessionAmProxy.getAMStatus(any(), any())).thenThrow(new ServiceException("error")); client.callRealGetSessionAMProxy = true; when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState()) - .thenReturn(YarnApplicationState.FAILED); + .thenReturn(YarnApplicationState.FAILED); Thread.sleep(3 * amHeartBeatTimeoutSecs * 1000); assertTrue(client.getAMKeepAliveService().isTerminated()); } diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java index 29a6769716..e818dc3638 100644 --- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java +++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -77,24 +77,26 @@ import org.apache.tez.serviceplugins.api.ServicePluginsDescriptor; import org.junit.Assert; import org.junit.Test; + /** - * + * */ public class TestTezClientUtils { private static String TEST_ROOT_DIR = "target" + Path.SEPARATOR + TestTezClientUtils.class.getName() + "-tmpDir"; private static final File STAGING_DIR = new File(System.getProperty("test.build.data", "target"), TestTezClientUtils.class.getName()).getAbsoluteFile(); + /** - * + * */ - @Test (timeout=5000) + @Test(timeout = 5000) public void validateSetTezJarLocalResourcesNotDefined() throws Exception { TezConfiguration conf = new TezConfiguration(false); Credentials credentials = new Credentials(); try { - Map resources = new HashMap(); + Map resources = new HashMap(); TezClientUtils.setupTezJarsLocalResources(conf, credentials, resources); Assert.fail("Expected TezUncheckedException"); } catch (TezUncheckedException e) { @@ -102,7 +104,7 @@ public void validateSetTezJarLocalResourcesNotDefined() throws Exception { } } - @Test (timeout=5000) + @Test(timeout = 5000) public void validateSetTezJarLocalResourcesDefinedButEmpty() throws Exception { File emptyDir = new File(TEST_ROOT_DIR, "emptyDir"); emptyDir.deleteOnExit(); @@ -111,7 +113,7 @@ public void validateSetTezJarLocalResourcesDefinedButEmpty() throws Exception { conf.set(TezConfiguration.TEZ_LIB_URIS, emptyDir.toURI().toURL().toString()); Credentials credentials = new Credentials(); try { - Map resources = new HashMap(); + Map resources = new HashMap(); TezClientUtils.setupTezJarsLocalResources(conf, credentials, resources); Assert.fail("Expected TezUncheckedException"); } catch (TezUncheckedException e) { @@ -120,15 +122,15 @@ public void validateSetTezJarLocalResourcesDefinedButEmpty() throws Exception { } /** - * + * */ - @Test(expected=FileNotFoundException.class, timeout=5000) + @Test(expected = FileNotFoundException.class, timeout = 5000) public void validateSetTezJarLocalResourcesDefinedNonExistingDirectory() throws Exception { TezConfiguration conf = new TezConfiguration(); conf.set(TezConfiguration.TEZ_LIB_URIS, "file:///foo"); Credentials credentials = new Credentials(); - Map resources = new HashMap(); + Map resources = new HashMap(); TezClientUtils.setupTezJarsLocalResources(conf, credentials, resources); } @@ -152,7 +154,7 @@ private static List getDirAndFileURL() throws MalformedURLException { return urls; } - @Test (timeout=20000) + @Test(timeout = 20000) public void validateSetTezJarLocalResourcesDefinedExistingDirectory() throws Exception { List cp = getDirAndFileURL(); StringBuffer buffer = new StringBuffer(); @@ -176,7 +178,7 @@ public void validateSetTezJarLocalResourcesDefinedExistingDirectory() throws Exc String[] firList = file.list(); for (String fileNme : firList) { File innerFile = new File(file, fileNme); - if (!innerFile.isDirectory()){ + if (!innerFile.isDirectory()) { assertTrue(resourceNames.contains(innerFile.getName())); assertedDir = true; } @@ -192,10 +194,10 @@ public void validateSetTezJarLocalResourcesDefinedExistingDirectory() throws Exc } /** - * + * * @throws Exception */ - @Test (timeout=5000) + @Test(timeout = 5000) public void validateSetTezJarLocalResourcesDefinedExistingDirectoryIgnored() throws Exception { List cp = getDirAndFileURL(); StringBuffer buffer = new StringBuffer(); @@ -213,10 +215,10 @@ public void validateSetTezJarLocalResourcesDefinedExistingDirectoryIgnored() thr } /** - * + * * @throws Exception */ - @Test (timeout=20000) + @Test(timeout = 20000) public void validateSetTezJarLocalResourcesDefinedExistingDirectoryIgnoredSetToFalse() throws Exception { List cp = getDirAndFileURL(); StringBuffer buffer = new StringBuffer(); @@ -233,11 +235,10 @@ public void validateSetTezJarLocalResourcesDefinedExistingDirectoryIgnoredSetToF assertFalse(localizedMap.isEmpty()); } - - /** + /** * */ - @Test (timeout=5000) + @Test(timeout = 5000) public void testTezDefaultFS() throws Exception { FileSystem localFs = FileSystem.getLocal(new Configuration()); StringBuilder tezLibUris = new StringBuilder(); @@ -264,10 +265,10 @@ public void testTezDefaultFS() throws Exception { Assert.assertTrue(localFs.delete(topDir, true)); } - /** + /** * */ - @Test (timeout=5000) + @Test(timeout = 5000) public void validateSetTezJarLocalResourcesMultipleTarballs() throws Exception { FileSystem localFs = FileSystem.getLocal(new Configuration()); StringBuilder tezLibUris = new StringBuilder(); @@ -299,16 +300,15 @@ public void validateSetTezJarLocalResourcesMultipleTarballs() throws Exception { Assert.assertFalse(resourceNames.contains("f1.tar.gz")); Assert.assertFalse(resourceNames.contains("f2.tar.gz")); - Assert.assertTrue(localFs.delete(tarFile1, true)); Assert.assertTrue(localFs.delete(tarFile2, true)); Assert.assertTrue(localFs.delete(topDir, true)); } - /** + /** * */ - @Test (timeout=5000) + @Test(timeout = 5000) public void validateSetTezJarLocalResourcesMixTarballAndJar() throws Exception { FileSystem localFs = FileSystem.getLocal(new Configuration()); StringBuilder tezLibUris = new StringBuilder(); @@ -374,7 +374,7 @@ public void testAppSubmissionContextForPriority() throws Exception { assertEquals(testpriority, appcontext.getPriority().getPriority()); } - @Test(timeout=1000) + @Test(timeout = 1000) // when tez config property for app priority not set, // tez should not set app priority and let YARN deal with it. public void testSetApplicationPriority() { @@ -386,7 +386,7 @@ public void testSetApplicationPriority() { assertNull(appContext.getPriority()); } - @Test(timeout=1000) + @Test(timeout = 1000) public void testSetApplicationTags() { TezConfiguration conf = new TezConfiguration(false); conf.set(TezConfiguration.TEZ_APPLICATION_TAGS, "foo,bar"); @@ -395,7 +395,7 @@ public void testSetApplicationTags() { .newRecord(ApplicationSubmissionContext.class); Collection tagsFromConf = amconfig.getTezConfiguration().getTrimmedStringCollection( - TezConfiguration.TEZ_APPLICATION_TAGS); + TezConfiguration.TEZ_APPLICATION_TAGS); appContext.setApplicationTags(new HashSet(tagsFromConf)); assertTrue(appContext.getApplicationTags().contains("foo")); assertTrue(appContext.getApplicationTags().contains("bar")); @@ -533,8 +533,8 @@ public void testAMCommandOpts() { String amOptsConstructed = TezClientUtils.constructAMLaunchOpts(tezConf, Resource.newInstance(1024, 1)); assertEquals(tmpOpts + " " - + TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT + " " - + amCommandOpts, + + TezConfiguration.TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT + " " + + amCommandOpts, amOptsConstructed); // Test2: Setup cluster-default command opts explicitly @@ -545,7 +545,6 @@ public void testAMCommandOpts() { TezClientUtils.constructAMLaunchOpts(tezConf, Resource.newInstance(1024, 1)); assertEquals(tmpOpts + " " + clusterDefaultCommandOpts + " " + amCommandOpts, amOptsConstructed); - // Test3: Don't setup Xmx explicitly final double factor = 0.8; amCommandOpts = "-Dtest.property"; @@ -615,8 +614,7 @@ public void testTaskCommandOpts() throws TezException { "]", taskOptsConstructed.startsWith(expected)); } - - @Test (timeout=5000) + @Test(timeout = 5000) public void testDefaultMemoryJavaOpts() { final double factor = 0.8; String origJavaOpts = "-Xmx"; @@ -671,7 +669,7 @@ public void testDefaultMemoryJavaOpts() { Assert.assertTrue(javaOpts.contains("-Xmx4000m")); } - @Test (timeout=5000) + @Test(timeout = 5000) public void testDefaultLoggingJavaOpts() { String origJavaOpts = null; String javaOpts = TezClientUtils.maybeAddDefaultLoggingJavaOpts("FOOBAR", origJavaOpts); @@ -692,9 +690,9 @@ public void testDefaultLoggingJavaOptsWithRootLogger() { && javaOpts.contains("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator")); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testConfSerializationForAm() { - Configuration conf =new Configuration(false); + Configuration conf = new Configuration(false); String val1 = "fixedProperty"; String val2 = "parametrizedProperty/${user.name}"; String expVal2 = "parametrizedProperty/" + System.getProperty("user.name"); @@ -788,7 +786,6 @@ public static void testLocalResourceVisibility(DistributedFileSystem remoteFs, C TezClientUtils.setupTezJarsLocalResources(tezConf, new Credentials(), lrMap); Assert.assertEquals(LocalResourceVisibility.PRIVATE, lrMap.get(TezConstants.TEZ_TAR_LR_NAME).getVisibility()); - } finally { if (topLevelDir != null) { remoteFs.delete(topLevelDir, true); @@ -827,7 +824,7 @@ private Path createFile(FileSystem fs, Path dir, String fileName) throws IOExcep return fs.makeQualified(f1); } - @Test (timeout=5000) + @Test(timeout = 5000) public void validateSetTezAuxLocalResourcesWithFilesAndFolders() throws Exception { FileSystem localFs = FileSystem.getLocal(new Configuration()); List resources = new ArrayList(); @@ -901,7 +898,6 @@ public void testTaskLaunchCmdOptsSetup() throws TezException { Assert.assertEquals(opts, TezConfiguration.TEZ_TASK_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT + " " + taskOpts + " " + vOpts); - } @Test(timeout = 5000) @@ -927,7 +923,6 @@ public void testClusterTaskLaunchCmdOptsSetup() throws TezException { opts = TezClientUtils.addDefaultsToTaskLaunchCmdOpts(vOpts, conf); Assert.assertEquals(opts, adminOpts + " " + taskOpts + " " + vOpts); - } @Test diff --git a/tez-api/src/test/java/org/apache/tez/common/TestJavaOptsChecker.java b/tez-api/src/test/java/org/apache/tez/common/TestJavaOptsChecker.java index 7f86e14f41..a6d246484f 100644 --- a/tez-api/src/test/java/org/apache/tez/common/TestJavaOptsChecker.java +++ b/tez-api/src/test/java/org/apache/tez/common/TestJavaOptsChecker.java @@ -85,7 +85,6 @@ public void testPositiveNegativeOpts() throws TezException { // Invalid negative can be ignored opts = " -XX:+UseG1GC -XX:+UseParallelGC -XX:-UseG1GC -XX:-UseConcMarkSweepGC "; javaOptsChecker.checkOpts(opts); - } @Test(timeout = 5000) @@ -107,8 +106,5 @@ public void testSpecialCaseNonConflictingGCOptions() throws TezException { Assert.assertTrue(e.getMessage(), e.getMessage().contains("Invalid/conflicting GC options found")); } - - } - } diff --git a/tez-api/src/test/java/org/apache/tez/common/TestRPCUtil.java b/tez-api/src/test/java/org/apache/tez/common/TestRPCUtil.java index 1e63b471ca..6c7f90eb67 100644 --- a/tez-api/src/test/java/org/apache/tez/common/TestRPCUtil.java +++ b/tez-api/src/test/java/org/apache/tez/common/TestRPCUtil.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -32,7 +32,7 @@ public class TestRPCUtil { - @Test (timeout=1000) + @Test(timeout = 1000) public void testUnknownExceptionUnwrapping() { Class exception = TezException.class; String className = "UnknownException.class"; @@ -56,7 +56,6 @@ public void testRemoteIOExceptionDerivativeUnwrapping() { public void testRemoteTezExceptionUnwrapping() { Class exception = TezException.class; verifyRemoteExceptionUnwrapping(exception, exception.getName()); - } @Test @@ -126,7 +125,7 @@ public void testRPCRuntimeExceptionUnwrapping() { Throwable t = null; try { RPCUtil.unwrapAndThrowException(se); - } catch (Throwable thrown) { + } catch (Throwable thrown) { t = thrown; } @@ -165,17 +164,14 @@ private void verifyRemoteExceptionUnwrapping( .getMessage().contains(message)); } - - @Test (timeout=1000) + @Test(timeout = 1000) public void testRemoteNonIOExceptionUnwrapping() { Class exception = TezException.class; verifyRemoteExceptionUnwrapping(exception, IOException.class.getName(), false); } - private static class TezTestExceptionNoConstructor extends Exception { private static final long serialVersionUID = 1L; } - } diff --git a/tez-api/src/test/java/org/apache/tez/common/TestReflectionUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestReflectionUtils.java index ed3814d000..cfcdaff1b0 100644 --- a/tez-api/src/test/java/org/apache/tez/common/TestReflectionUtils.java +++ b/tez-api/src/test/java/org/apache/tez/common/TestReflectionUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -39,6 +39,7 @@ public class TestReflectionUtils { public static class ParameterizedConstructorClass { final String first; final int second; + public ParameterizedConstructorClass(String first, int second) { this.first = first; this.second = second; @@ -46,10 +47,9 @@ public ParameterizedConstructorClass(String first, int second) { } @Test(timeout = 5000) - public void testConstructorWithParameters() throws TezReflectionException - { - Class[] parameterTypes = new Class[] { String.class, Integer.TYPE }; - Object[] parameters = new Object[] { new String("test"), 1 }; + public void testConstructorWithParameters() throws TezReflectionException { + Class[] parameterTypes = new Class[]{String.class, Integer.TYPE}; + Object[] parameters = new Object[]{new String("test"), 1}; ParameterizedConstructorClass instance = ReflectionUtils.createClazzInstance( ParameterizedConstructorClass.class.getName(), parameterTypes, parameters); assertEquals("Class not constructed with first parameter correctly", instance.first, "test"); diff --git a/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java index d5dc6fd6b5..16bd65cbde 100644 --- a/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java +++ b/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -49,7 +49,8 @@ public class TestTezCommonUtils { private static final File LOCAL_STAGING_DIR = new File(System.getProperty("test.build.data"), TestTezCommonUtils.class.getSimpleName()).getAbsoluteFile(); private static String RESOLVED_STAGE_DIR; - private static Configuration conf = new Configuration();; + private static Configuration conf = new Configuration(); + ; private static String TEST_ROOT_DIR = "target" + Path.SEPARATOR + TestTezCommonUtils.class.getName() + "-tmpDir"; private static MiniDFSCluster dfsCluster = null; @@ -248,27 +249,25 @@ public void testLocalResourceVisibility() throws Exception { @Test(timeout = 5000) public void testStringTokenize() { String s = "foo:bar:xyz::too"; - String[] expectedTokens = { "foo", "bar" , "xyz" , "too"}; + String[] expectedTokens = {"foo", "bar", "xyz", "too"}; String[] tokens = new String[4]; TezCommonUtils.tokenizeString(s, ":").toArray(tokens); Assert.assertArrayEquals(expectedTokens, tokens); } - @Test(timeout = 5000) public void testAddAdditionalLocalResources() { String lrName = "LR"; Map originalLrs; - originalLrs= Maps.newHashMap(); + originalLrs = Maps.newHashMap(); originalLrs.put(lrName, LocalResource.newInstance( URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); - Map additionalLrs; // Same path, same size. - originalLrs= Maps.newHashMap(); + originalLrs = Maps.newHashMap(); originalLrs.put(lrName, LocalResource.newInstance( URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); @@ -278,7 +277,7 @@ public void testAddAdditionalLocalResources() { TezCommonUtils.addAdditionalLocalResources(additionalLrs, originalLrs, ""); // Same path, different size. - originalLrs= Maps.newHashMap(); + originalLrs = Maps.newHashMap(); originalLrs.put(lrName, LocalResource.newInstance( URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); @@ -293,7 +292,7 @@ public void testAddAdditionalLocalResources() { } // Different path, same size, diff timestamp - originalLrs= Maps.newHashMap(); + originalLrs = Maps.newHashMap(); originalLrs.put(lrName, LocalResource.newInstance( URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); @@ -303,7 +302,7 @@ public void testAddAdditionalLocalResources() { TezCommonUtils.addAdditionalLocalResources(additionalLrs, originalLrs, ""); // Different path, different size - originalLrs= Maps.newHashMap(); + originalLrs = Maps.newHashMap(); originalLrs.put(lrName, LocalResource.newInstance( URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); @@ -316,10 +315,9 @@ public void testAddAdditionalLocalResources() { } catch (TezUncheckedException e) { Assert.assertTrue(e.getMessage().contains("Duplicate Resources found with different size")); } - } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testAMClientHeartBeatTimeout() { TezConfiguration conf = new TezConfiguration(false); @@ -363,8 +361,6 @@ public void testAMClientHeartBeatTimeout() { conf.setInt(TezConfiguration.TEZ_AM_CLIENT_HEARTBEAT_POLL_INTERVAL_MILLIS, 2000); Assert.assertEquals(2000, TezCommonUtils.getAMClientHeartBeatPollIntervalMillis(conf, 20000, 5)); - - } @Test @@ -373,7 +369,7 @@ public void testLogSystemProperties() throws Exception { // test default logging conf.set(TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG, " "); String value = TezCommonUtils.getSystemPropertiesToLog(conf); - for(String key: TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG_DEFAULT) { + for (String key : TezConfiguration.TEZ_JVM_SYSTEM_PROPERTIES_TO_LOG_DEFAULT) { Assert.assertTrue(value.contains(key)); } @@ -389,16 +385,16 @@ public void testLogSystemProperties() throws Exception { Assert.assertFalse(value.contains(version)); } - @Test(timeout=5000) + @Test(timeout = 5000) public void testGetDAGSessionTimeout() { Configuration conf = new Configuration(false); - Assert.assertEquals(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS_DEFAULT*1000, + Assert.assertEquals(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS_DEFAULT * 1000, TezCommonUtils.getDAGSessionTimeout(conf)); // set to 1 month - * 1000 guaranteed to cross positive integer boundary conf.setInt(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS, 24 * 60 * 60 * 30); - Assert.assertEquals(86400l*1000*30, + Assert.assertEquals(86400l * 1000 * 30, TezCommonUtils.getDAGSessionTimeout(conf)); // set to negative val @@ -410,10 +406,8 @@ public void testGetDAGSessionTimeout() { conf.setInt(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS, 0); Assert.assertEquals(1000, TezCommonUtils.getDAGSessionTimeout(conf)); - } - @Test public void testMkDirForAM() throws IOException { Configuration remoteConf = new Configuration(); diff --git a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java index 4fdd61be9a..b7a5cbbd1e 100644 --- a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java +++ b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -112,7 +112,7 @@ public void testSetupDefaultEnvironment() { String value3 = environment.get("LD_LIBRARY_PATH"); Assert.assertEquals("User env should append default env", Environment.PWD.$() + File.pathSeparator + "USER_PATH" + File.pathSeparator + "DEFAULT_PATH", value3); - } + } @Test(timeout = 5000) public void testTezLibUrisClasspath() { @@ -130,7 +130,7 @@ public void testSetEnvFromInputString() { Map environment = new HashMap(); environment.put("JAVA_HOME", "/path/jdk"); String goodEnv = "a1=1,b_2=2,_c=3,d=4,e=,f_win=%JAVA_HOME%" - + ",g_nix=$JAVA_HOME"; + + ",g_nix=$JAVA_HOME"; appendToEnvFromInputString(environment, goodEnv, File.pathSeparator); assertEquals("1", environment.get("a1")); assertEquals("2", environment.get("b_2")); @@ -152,7 +152,7 @@ public void testSetEnvFromInputString() { // Test "=" in the value part environment.clear(); appendToEnvFromInputString(environment, "b1,e1==,e2=a1=a2,b2", - File.pathSeparator); + File.pathSeparator); assertEquals("=", environment.get("e1")); assertEquals("a1=a2", environment.get("e2")); } diff --git a/tez-api/src/test/java/org/apache/tez/common/TestVersionInfo.java b/tez-api/src/test/java/org/apache/tez/common/TestVersionInfo.java index 67f97b86a5..212c3895ff 100644 --- a/tez-api/src/test/java/org/apache/tez/common/TestVersionInfo.java +++ b/tez-api/src/test/java/org/apache/tez/common/TestVersionInfo.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -59,5 +59,4 @@ public void testTest3File() { public void testNonExistentFile() { VersionInfo versionInfo = new VersionInfo("test4"); } - } diff --git a/tez-api/src/test/java/org/apache/tez/common/security/TestACLConfigurationParser.java b/tez-api/src/test/java/org/apache/tez/common/security/TestACLConfigurationParser.java index f1a2c49485..55c18b6dbf 100644 --- a/tez-api/src/test/java/org/apache/tez/common/security/TestACLConfigurationParser.java +++ b/tez-api/src/test/java/org/apache/tez/common/security/TestACLConfigurationParser.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -146,5 +146,4 @@ public void testDAGACLConfigParser() { Assert.assertTrue(parser.getAllowedGroups().get(ACLType.YARN_ADMIN_ACL).contains("admgrp4")); Assert.assertTrue(parser.getAllowedGroups().get(ACLType.YARN_ADMIN_ACL).contains("admgrp5")); } - } diff --git a/tez-api/src/test/java/org/apache/tez/common/security/TestACLManager.java b/tez-api/src/test/java/org/apache/tez/common/security/TestACLManager.java index fc9c24e108..eb28c55720 100644 --- a/tez-api/src/test/java/org/apache/tez/common/security/TestACLManager.java +++ b/tez-api/src/test/java/org/apache/tez/common/security/TestACLManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -30,7 +30,6 @@ import org.junit.Assert; import org.junit.Test; - public class TestACLManager { private static final String[] noGroups = new String[0]; @@ -76,9 +75,9 @@ public void testCurrentUserACLChecks() { @Test(timeout = 5000) public void testOtherUserACLChecks() throws IOException { - String[] groups1 = new String[] {"grp1", "grp2"}; - String[] groups2 = new String[] {"grp3", "grp4"}; - String[] groups3 = new String[] {"grp5", "grp6"}; + String[] groups1 = new String[]{"grp1", "grp2"}; + String[] groups2 = new String[]{"grp3", "grp4"}; + String[] groups3 = new String[]{"grp5", "grp6"}; UserGroupInformation currentUser = UserGroupInformation.createUserForTesting("currentUser", noGroups); UserGroupInformation user1 = UserGroupInformation.createUserForTesting("user1", groups1); // belongs to grp1 and grp2 @@ -104,7 +103,7 @@ public void testOtherUserACLChecks() throws IOException { Assert.assertTrue(aclManager.checkAccess(user2, ACLType.AM_VIEW_ACL)); Assert.assertFalse(aclManager.checkAccess(user3, ACLType.AM_VIEW_ACL)); Assert.assertTrue(aclManager.checkAccess(user4, ACLType.AM_VIEW_ACL)); - Assert.assertFalse(aclManager.checkAccess(user5, ACLType.AM_VIEW_ACL)); + Assert.assertFalse(aclManager.checkAccess(user5, ACLType.AM_VIEW_ACL)); Assert.assertFalse(aclManager.checkAccess(user6, ACLType.AM_VIEW_ACL)); Assert.assertTrue(aclManager.checkAccess(currentUser, ACLType.AM_MODIFY_ACL)); @@ -118,9 +117,9 @@ public void testOtherUserACLChecks() throws IOException { @Test(timeout = 5000) public void testNoGroupsACLChecks() throws IOException { - String[] groups1 = new String[] {"grp1", "grp2"}; - String[] groups2 = new String[] {"grp3", "grp4"}; - String[] groups3 = new String[] {"grp5", "grp6"}; + String[] groups1 = new String[]{"grp1", "grp2"}; + String[] groups2 = new String[]{"grp3", "grp4"}; + String[] groups3 = new String[]{"grp5", "grp6"}; UserGroupInformation currentUser = UserGroupInformation.createUserForTesting("currentUser", noGroups); UserGroupInformation user1 = UserGroupInformation.createUserForTesting("user1", groups1); // belongs to grp1 and grp2 @@ -158,10 +157,10 @@ public void testNoGroupsACLChecks() throws IOException { @Test(timeout = 5000) public void checkAMACLs() throws IOException { - String[] groups1 = new String[] {"grp1", "grp2"}; - String[] groups2 = new String[] {"grp3", "grp4"}; - String[] groups3 = new String[] {"grp5", "grp6"}; - String[] admingroup1 = new String[] {"admgrp1"}; + String[] groups1 = new String[]{"grp1", "grp2"}; + String[] groups2 = new String[]{"grp3", "grp4"}; + String[] groups3 = new String[]{"grp5", "grp6"}; + String[] admingroup1 = new String[]{"admgrp1"}; UserGroupInformation currentUser = UserGroupInformation.createUserForTesting("currentUser", noGroups); UserGroupInformation user1 = UserGroupInformation.createUserForTesting("user1", groups1); // belongs to grp1 and grp2 @@ -229,10 +228,10 @@ public void checkAMACLs() throws IOException { @Test(timeout = 5000) public void checkDAGACLs() throws IOException { - String[] groups1 = new String[] {"grp1", "grp2"}; - String[] groups2 = new String[] {"grp3", "grp4"}; - String[] groups3 = new String[] {"grp5", "grp6"}; - String[] admingroup1 = new String[] {"admgrp1"}; + String[] groups1 = new String[]{"grp1", "grp2"}; + String[] groups2 = new String[]{"grp3", "grp4"}; + String[] groups3 = new String[]{"grp5", "grp6"}; + String[] admingroup1 = new String[]{"admgrp1"}; UserGroupInformation currentUser = UserGroupInformation.createUserForTesting("currentUser", noGroups); UserGroupInformation user1 = UserGroupInformation.createUserForTesting("user1", groups1); // belongs to grp1 and grp2 @@ -313,7 +312,6 @@ public void checkDAGACLs() throws IOException { Assert.assertTrue(aclManager.checkDAGModifyAccess(user6)); Assert.assertTrue(aclManager.checkDAGModifyAccess(admuser1)); Assert.assertTrue(aclManager.checkDAGModifyAccess(admuser2)); - } @Test(timeout = 5000) @@ -414,7 +412,5 @@ public void testConvertToYARNACLs() { yarnAcls = aclManager1.toYARNACls(); Assert.assertEquals("c1 grp3,grp4", yarnAcls.get(ApplicationAccessType.VIEW_APP)); - } - } diff --git a/tez-api/src/test/java/org/apache/tez/common/security/TestDAGAccessControls.java b/tez-api/src/test/java/org/apache/tez/common/security/TestDAGAccessControls.java index 4335a20fd3..7fc29b6f30 100644 --- a/tez-api/src/test/java/org/apache/tez/common/security/TestDAGAccessControls.java +++ b/tez-api/src/test/java/org/apache/tez/common/security/TestDAGAccessControls.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -42,8 +42,7 @@ public void testStringBasedConstructor() { Assert.assertTrue(dagAccessControls.getGroupsWithModifyACLs().contains("g2")); } - - @Test(timeout=5000) + @Test(timeout = 5000) public void testMergeIntoAmAcls() { DAGAccessControls dagAccessControls = new DAGAccessControls("u1 g1", "u2 g2"); Configuration conf = new Configuration(false); @@ -129,8 +128,8 @@ public void assertACLS(String expected, String obtained) { return; } - String [] parts1 = expected.split(" "); - String [] parts2 = obtained.split(" "); + String[] parts1 = expected.split(" "); + String[] parts2 = obtained.split(" "); Assert.assertEquals(parts1.length, parts2.length); diff --git a/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java b/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java index 4c144174c7..22723ef0df 100644 --- a/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java +++ b/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -67,7 +67,7 @@ public void testBinaryCredentials() throws Exception { try { Path TEST_ROOT_DIR = new Path("target"); binaryTokenFile = FileSystem.getLocal(conf).makeQualified( - new Path(TEST_ROOT_DIR, "tokenFile")).toUri().getPath(); + new Path(TEST_ROOT_DIR, "tokenFile")).toUri().getPath(); MockFileSystem fs1 = createFileSystemForServiceName("service1"); MockFileSystem fs2 = createFileSystemForServiceName("service2"); @@ -80,7 +80,6 @@ public void testBinaryCredentials() throws Exception { creds.addToken(token2.getService(), token2); creds.writeTokenStorageFile(new Path(binaryTokenFile), conf); - Credentials newCreds = new Credentials(); TokenCache.mergeBinaryTokens(newCreds, conf, binaryTokenFile); @@ -97,7 +96,7 @@ public void testBinaryCredentials() throws Exception { } } - @Test(timeout=5000) + @Test(timeout = 5000) public void testObtainTokensForFileSystems() throws Exception { Path[] paths = makePaths(100, "test://dir/file"); Credentials creds = new Credentials(); @@ -130,6 +129,7 @@ private Path[] makePaths(int count, String prefix) throws Exception { public static class TestFileSystem extends FilterFileSystem { static final FileSystem fs = mock(FileSystem.class); + static { try { when(fs.getUri()).thenReturn(new URI("test://dir")); @@ -156,6 +156,7 @@ private MockFileSystem createFileSystemForServiceName(final String service) when(mockFs.getDelegationToken(any())).thenAnswer( new Answer>() { int unique = 0; + @Override public Token answer(InvocationOnMock invocation) throws Throwable { Token token = new Token(); @@ -178,5 +179,4 @@ private void checkTokens(Credentials creds, Credentials newCreds) { Assert.assertEquals(token, credsToken); } } - } diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java index 05c4e30cf0..596b4baa8e 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAG.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -108,7 +108,6 @@ public void testDuplicatedVertexGroup() { } // it is possible to create vertex group with same member but different group name dag.createVertexGroup("group_2", v1, v2); - } @Test(timeout = 5000) @@ -162,11 +161,11 @@ public void testDAGConf() { // set invalid AM level configuration try { - dag.setConf(TezConfiguration.TEZ_AM_SESSION_MODE, true+""); + dag.setConf(TezConfiguration.TEZ_AM_SESSION_MODE, true + ""); Assert.fail(); } catch (IllegalStateException e) { Assert.assertEquals("tez.am.mode.session is set at the scope of DAG," - + " but it is only valid in the scope of AM", + + " but it is only valid in the scope of AM", e.getMessage()); } // set valid DAG level configuration @@ -183,11 +182,11 @@ public void testVertexConf() { // set invalid AM level configuration try { - v1.setConf(TezConfiguration.TEZ_AM_SESSION_MODE, true+""); + v1.setConf(TezConfiguration.TEZ_AM_SESSION_MODE, true + ""); Assert.fail(); } catch (IllegalStateException e) { Assert.assertEquals("tez.am.mode.session is set at the scope of VERTEX," - + " but it is only valid in the scope of AM", + + " but it is only valid in the scope of AM", e.getMessage()); } @@ -197,7 +196,7 @@ public void testVertexConf() { Assert.fail("should fail due to invalid configuration set"); } catch (IllegalStateException e) { Assert.assertEquals("tez.am.commit-all-outputs-on-dag-success is set at the scope of VERTEX," - + " but it is only valid in the scope of DAG", + + " but it is only valid in the scope of DAG", e.getMessage()); } // set valid Vertex level configuration @@ -309,7 +308,7 @@ public void testDuplicatedOutput_2() { v1.addDataSink("output_1", dataSink); Vertex v2 = Vertex.create("v1", ProcessorDescriptor.create("dummyProcessor")); - VertexGroup vGroup = dag.createVertexGroup("group_1", v1,v2); + VertexGroup vGroup = dag.createVertexGroup("group_1", v1, v2); try { vGroup.addDataSink("output_1", DataSinkDescriptor.create(OutputDescriptor.create("dummyOutput"), null, null)); @@ -340,7 +339,6 @@ public void testCallerContext() { CallerContext callerContext = CallerContext.create("ctxt", "desc"); Assert.assertTrue(callerContext.toString().contains("desc")); Assert.assertFalse(callerContext.contextAsSimpleString().contains("desc")); - } @Test @@ -359,8 +357,8 @@ public void testRecreateDAG() { TezConfiguration tezConf = new TezConfiguration(); DAGPlan firstPlan = dag.createDag(tezConf, null, null, null, false); for (int i = 0; i < 3; ++i) { - DAGPlan dagPlan = dag.createDag(tezConf, null, null, null, false); - Assert.assertEquals(dagPlan, firstPlan); + DAGPlan dagPlan = dag.createDag(tezConf, null, null, null, false); + Assert.assertEquals(dagPlan, firstPlan); } } @@ -397,7 +395,7 @@ public void testCreateDAGForHistoryLogLevel() { Assert.fail("Expected illegal argument exception"); } catch (IllegalArgumentException e) { Assert.assertEquals("Config: " + TezConfiguration.TEZ_HISTORY_LOGGING_LOGLEVEL + - " is set to invalid value: invalid", e.getMessage()); + " is set to invalid value: invalid", e.getMessage()); } // Set value in dag, should override tez conf value. @@ -419,7 +417,7 @@ public void testCreateDAGForHistoryLogLevel() { Assert.fail("Expected illegal argument exception"); } catch (IllegalArgumentException e) { Assert.assertEquals("Config: " + TezConfiguration.TEZ_HISTORY_LOGGING_LOGLEVEL + - " is set to invalid value: invalid", e.getMessage()); + " is set to invalid value: invalid", e.getMessage()); } } diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java index 8e1011feb7..eafd775544 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,6 +19,7 @@ package org.apache.tez.dag.api; import java.nio.ByteBuffer; + import static org.junit.Assert.*; import java.io.File; @@ -64,45 +65,43 @@ public class TestDAGPlan { public void testBasicJobPlanSerde() throws IOException { DAGPlan job = DAGPlan.newBuilder() - .setName("test") - .addVertex( - VertexPlan.newBuilder() - .setName("vertex1") - .setType(PlanVertexType.NORMAL) - .addTaskLocationHint(PlanTaskLocationHint.newBuilder().addHost("machineName").addRack("rack1").build()) - .setTaskConfig( - PlanTaskConfiguration.newBuilder() - .setNumTasks(2) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("x.y") - .build()) - .build()) + .setName("test") + .addVertex( + VertexPlan.newBuilder() + .setName("vertex1") + .setType(PlanVertexType.NORMAL) + .addTaskLocationHint(PlanTaskLocationHint.newBuilder().addHost("machineName").addRack("rack1").build()) + .setTaskConfig( + PlanTaskConfiguration.newBuilder() + .setNumTasks(2) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("x.y") + .build()) + .build()) .build(); - File file = tempFolder.newFile("jobPlan"); - FileOutputStream outStream = null; - try { - outStream = new FileOutputStream(file); - job.writeTo(outStream); - } - finally { - if(outStream != null){ - outStream.close(); - } - } - - DAGPlan inJob; - FileInputStream inputStream; - try { - inputStream = new FileInputStream(file); - inJob = DAGPlan.newBuilder().mergeFrom(inputStream).build(); - } - finally { - outStream.close(); - } - - Assert.assertEquals(job, inJob); + File file = tempFolder.newFile("jobPlan"); + FileOutputStream outStream = null; + try { + outStream = new FileOutputStream(file); + job.writeTo(outStream); + } finally { + if (outStream != null) { + outStream.close(); + } + } + + DAGPlan inJob; + FileInputStream inputStream; + try { + inputStream = new FileInputStream(file); + inJob = DAGPlan.newBuilder().mergeFrom(inputStream).build(); + } finally { + outStream.close(); + } + + Assert.assertEquals(job, inJob); } @Test(timeout = 5000) @@ -276,7 +275,7 @@ public void userVertexOrderingIsMaintained() { assertEquals("output", edgeProperty.getEdgeSource().getClassName()); } - @Test (timeout=5000) + @Test(timeout = 5000) public void testCredentialsSerde() { DAG dag = DAG.create("testDag"); ProcessorDescriptor pd1 = ProcessorDescriptor.create("processor1"). @@ -305,16 +304,16 @@ public void testCredentialsSerde() { Token token2 = new Token(); dagCredentials.addToken(new Text("Token1"), token1); dagCredentials.addToken(new Text("Token2"), token2); - + dag.setCredentials(dagCredentials); DAGPlan dagProto = dag.createDag(new TezConfiguration(), null, null, null, true); assertTrue(dagProto.hasCredentialsBinary()); - + Credentials fetchedCredentials = DagTypeConverters.convertByteStringToCredentials(dagProto .getCredentialsBinary()); - + assertEquals(2, fetchedCredentials.numberOfTokens()); assertNotNull(fetchedCredentials.getToken(new Text("Token1"))); assertNotNull(fetchedCredentials.getToken(new Text("Token2"))); @@ -342,7 +341,6 @@ public void testInvalidExecContext_1() { } catch (IllegalStateException e) { assertTrue(e.getMessage().contains("container execution")); } - } @Test(timeout = 5000) @@ -363,7 +361,6 @@ public void testInvalidExecContext_2() { VertexExecutionContext invalidExecContext3 = VertexExecutionContext.create("plugin", "plugin", "invalidplugin"); - DAG dag = DAG.create("dag1"); dag.setExecutionContext(VertexExecutionContext.createExecuteInContainers(true)); Vertex v1 = Vertex.create("testvertex", ProcessorDescriptor.create("processor1"), 1); @@ -373,7 +370,6 @@ public void testInvalidExecContext_2() { dag.createDag(new TezConfiguration(false), null, null, null, true, servicePluginsDescriptor, null); - // Set execute in AM should fail v1.setExecutionContext(VertexExecutionContext.createExecuteInAm(true)); try { @@ -424,7 +420,6 @@ public void testInvalidExecContext_2() { assertTrue(e.getMessage().contains("task communicator")); assertTrue(e.getMessage().contains("invalidplugin")); } - } @Test(timeout = 5000) @@ -516,7 +511,5 @@ public void testInvalidJavaOpts() { // Should not fail as no checker enabled conf.set(TezConfiguration.TEZ_TASK_LAUNCH_CMD_OPTS, " -XX:+UseParallelGC "); dag.createDag(conf, null, null, null, true, null, null); - } - } diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGVerify.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGVerify.java index bde4622d9b..746eb5f3a0 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGVerify.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGVerify.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -364,7 +364,7 @@ public void testVerify4() { // v3 v4 @Test(timeout = 5000) public void testCycle1() { - IllegalStateException ex=null; + IllegalStateException ex = null; Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create("MapProcessor"), dummyTaskCount, dummyTaskResource); @@ -406,10 +406,9 @@ public void testCycle1() { dag.addEdge(e2); dag.addEdge(e3); dag.addEdge(e4); - try{ + try { dag.verify(); - } - catch (IllegalStateException e){ + } catch (IllegalStateException e) { ex = e; } Assert.assertNotNull(ex); @@ -424,7 +423,7 @@ public void testCycle1() { // v3 v4 @Test(timeout = 5000) public void testCycle2() { - IllegalStateException ex=null; + IllegalStateException ex = null; Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create("MapProcessor"), dummyTaskCount, dummyTaskResource); @@ -466,10 +465,9 @@ public void testCycle2() { dag.addEdge(e2); dag.addEdge(e3); dag.addEdge(e4); - try{ + try { dag.verify(); - } - catch (IllegalStateException e){ + } catch (IllegalStateException e) { ex = e; } Assert.assertNotNull(ex); @@ -479,8 +477,8 @@ public void testCycle2() { // v1 -> v1 @Test(timeout = 5000) - public void testSelfCycle(){ - IllegalStateException ex=null; + public void testSelfCycle() { + IllegalStateException ex = null; Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create("MapProcessor"), dummyTaskCount, dummyTaskResource); @@ -492,10 +490,9 @@ public void testSelfCycle(){ DAG dag = DAG.create("testDag"); dag.addVertex(v1); dag.addEdge(e1); - try{ + try { dag.verify(); - } - catch (IllegalStateException e){ + } catch (IllegalStateException e) { ex = e; } Assert.assertNotNull(ex); @@ -505,7 +502,7 @@ public void testSelfCycle(){ @Test(timeout = 5000) public void repeatedVertexName() { - IllegalStateException ex=null; + IllegalStateException ex = null; Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create("MapProcessor"), dummyTaskCount, dummyTaskResource); @@ -517,8 +514,7 @@ public void repeatedVertexName() { dag.addVertex(v1); dag.addVertex(v1repeat); dag.verify(); - } - catch (IllegalStateException e){ + } catch (IllegalStateException e) { ex = e; } Assert.assertNotNull(ex); @@ -682,7 +678,7 @@ public void testVertexGroupWithMultipleOutputEdges() { dag.addEdge(e1); dag.addEdge(e2); dag.verify(); - for (int i = 0; i< 10;++i){ + for (int i = 0; i < 10; ++i) { dag.verify(); // should be OK when called multiple times } @@ -741,7 +737,7 @@ public void testVertexGroup() { dag.addVertex(v5); dag.addEdge(e1); dag.addEdge(e2); - for (int i = 0; i< 10;++i){ + for (int i = 0; i < 10; ++i) { dag.verify(); // should be OK when called multiple times } @@ -823,7 +819,7 @@ public void testVertexGroupOneToOne() { dag.addVertex(v5); dag.addEdge(e1); dag.addEdge(e2); - for (int i = 0; i< 10;++i){ + for (int i = 0; i < 10; ++i) { dag.verify(); // should be OK when called multiple times } @@ -863,8 +859,7 @@ public void BinaryOutput() { dag.addEdge(e1); dag.addEdge(e2); dag.verify(); - } - catch (IllegalStateException e){ + } catch (IllegalStateException e) { ex = e; } Assert.assertNull(ex); @@ -872,12 +867,11 @@ public void BinaryOutput() { @Test(timeout = 5000) public void testDagWithNoVertices() { - IllegalStateException ex=null; + IllegalStateException ex = null; try { DAG dag = DAG.create("testDag"); dag.verify(); - } - catch (IllegalStateException e){ + } catch (IllegalStateException e) { ex = e; } Assert.assertNotNull(ex); @@ -942,7 +936,6 @@ public void testMultipleRootInputsAllowed() { dag.createDag(new TezConfiguration(), null, null, null, true); } - @Test(timeout = 5000) public void testDAGCreateDataInference() { Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create(dummyProcessorClassName)); @@ -1113,7 +1106,7 @@ public void testDAGInvalidParallelism3() { v0.addDataSource("input", dsDesc); dag.addVertex(v0); dag.addEdge(Edge.create(v0, v1, EdgeProperty.create(DataMovementType.ONE_TO_ONE, - DataSourceType.PERSISTED,SchedulingType.SEQUENTIAL, + DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create(dummyOutputClassName), InputDescriptor.create(dummyInputClassName)))); dag.verify(); @@ -1138,7 +1131,7 @@ public void testDAGInvalidParallelism4() { v0.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(dummyVMPluginClassName)); dag.addVertex(v0); dag.addEdge(Edge.create(v0, v1, EdgeProperty.create(DataMovementType.ONE_TO_ONE, - DataSourceType.PERSISTED,SchedulingType.SEQUENTIAL, + DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create(dummyOutputClassName), InputDescriptor.create(dummyInputClassName)))); dag.verify(); @@ -1174,13 +1167,13 @@ public void testDAGWithConflictingResource() { Map localResourceMap = new HashMap<>(); String commonResourceKey = "local resource"; localResourceMap.put("lr", LocalResource.newInstance(null, LocalResourceType.FILE, - LocalResourceVisibility.APPLICATION, 0, 0)); + LocalResourceVisibility.APPLICATION, 0, 0)); dag.addTaskLocalFiles(localResourceMap); Vertex v1 = Vertex.create("v", ProcessorDescriptor.create(dummyProcessorClassName), 1); // same key but different resource localResourceMap.put("lr", LocalResource.newInstance(null, LocalResourceType.FILE, - LocalResourceVisibility.APPLICATION, 10, 0)); + LocalResourceVisibility.APPLICATION, 10, 0)); v1.addTaskLocalFiles(localResourceMap); dag.addVertex(v1); diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestDagTypeConverters.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestDagTypeConverters.java index edb7fd8445..a4c158b6a0 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/TestDagTypeConverters.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestDagTypeConverters.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -58,7 +58,7 @@ public void testTezEntityDescriptorSerialization() throws IOException { String historytext = "Bar123"; EntityDescriptor entityDescriptor = InputDescriptor.create("inputClazz").setUserPayload(payload) - .setHistoryText(historytext); + .setHistoryText(historytext); TezEntityDescriptorProto proto = DagTypeConverters.convertToDAGPlan(entityDescriptor); Assert.assertEquals(payload.getVersion(), proto.getTezUserPayload().getVersion()); @@ -93,7 +93,6 @@ public void testYarnPathTranslation() { Assert.assertEquals("/file", lr1UrlDeserialized.getFile()); Assert.assertEquals("hdfs", lr1UrlDeserialized.getScheme()); - // With port String p2String = "hdfs://mycluster:2311/file"; Path p2Path = new Path(p2String); @@ -109,14 +108,12 @@ public void testYarnPathTranslation() { Assert.assertEquals(2311, lr2UrlDeserialized.getPort()); } - @Test(timeout = 5000) public void testVertexExecutionContextTranslation() { VertexExecutionContext originalContext; VertexExecutionContextProto contextProto; VertexExecutionContext retrievedContext; - // Uber originalContext = VertexExecutionContext.createExecuteInAm(true); contextProto = DagTypeConverters.convertToProto(originalContext); @@ -136,7 +133,6 @@ public void testVertexExecutionContextTranslation() { assertEquals(originalContext, retrievedContext); } - static final String testScheduler = "testScheduler"; static final String testLauncher = "testLauncher"; static final String testComm = "testComm"; @@ -145,7 +141,6 @@ public void testVertexExecutionContextTranslation() { @Test(timeout = 5000) public void testServiceDescriptorTranslation() { - TaskSchedulerDescriptor[] taskSchedulers; ContainerLauncherDescriptor[] containerLaunchers; TaskCommunicatorDescriptor[] taskComms; @@ -180,7 +175,6 @@ public void testServiceDescriptorTranslation() { verifyPlugins(proto.getContainerLaunchersList(), 1, testLauncher, false); verifyPlugins(proto.getTaskCommunicatorsList(), 1, testComm, true); - // Multiple plugin set specified. All with a payload taskSchedulers = createTaskScheduelrs(3, true); containerLaunchers = createContainerLaunchers(3, true); diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestEntityDescriptor.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestEntityDescriptor.java index 82c22ad4ef..f805c877f1 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/TestEntityDescriptor.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestEntityDescriptor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -39,7 +39,7 @@ public class TestEntityDescriptor { public void verifyResults(InputDescriptor entityDescriptor, InputDescriptor deserialized, UserPayload payload, - String confVal) throws IOException { + String confVal) throws IOException { Assert.assertEquals(entityDescriptor.getClassName(), deserialized.getClassName()); // History text is not serialized when sending to tasks Assert.assertNull(deserialized.getHistoryText()); @@ -73,9 +73,9 @@ public void testSegmentedWrite(InputDescriptor entityDescriptor, InputDescriptor verifyResults(entityDescriptor, deserialized, payload, confVal); } - @Test (timeout=3000) + @Test(timeout = 3000) public void testEntityDescriptorHadoopSerialization() throws IOException { - /* This tests the alternate serialization code path + /* This tests the alternate serialization code path * if the DataOutput is not DataOutputBuffer * AND, if it indeed is, with a read/write payload */ Configuration conf = new Configuration(true); @@ -86,15 +86,15 @@ public void testEntityDescriptorHadoopSerialization() throws IOException { InputDescriptor deserialized = InputDescriptor.create("dummy"); InputDescriptor entityDescriptor = InputDescriptor.create("inputClazz").setUserPayload(payload) - .setHistoryText("Bar123"); + .setHistoryText("Bar123"); InputDescriptor entityDescriptorLivingInFear = spy(entityDescriptor); testSingularWrite(entityDescriptorLivingInFear, deserialized, payload, confVal); /* make read-only payload */ - payload = UserPayload.create(payload.getPayload()); + payload = UserPayload.create(payload.getPayload()); entityDescriptor = InputDescriptor.create("inputClazz").setUserPayload(payload) - .setHistoryText("Bar123"); + .setHistoryText("Bar123"); entityDescriptorLivingInFear = spy(entityDescriptor); testSegmentedWrite(entityDescriptorLivingInFear, deserialized, payload, confVal); } diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestHistoryLogLevel.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestHistoryLogLevel.java index 76c944d693..c500a463ea 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/TestHistoryLogLevel.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestHistoryLogLevel.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestTaskLocationHint.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestTaskLocationHint.java index d52e1942c9..5aa54549f7 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/TestTaskLocationHint.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestTaskLocationHint.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,8 +24,8 @@ import com.google.common.collect.Sets; public class TestTaskLocationHint { - - @Test (timeout = 5000) + + @Test(timeout = 5000) public void testEquality() { TaskLocationHint t1 = TaskLocationHint.createTaskLocationHint("v1", 0); TaskLocationHint t2 = TaskLocationHint.createTaskLocationHint("v1", 0); @@ -34,13 +34,13 @@ public void testEquality() { TaskLocationHint t5 = TaskLocationHint.createTaskLocationHint(null, null); TaskLocationHint t6 = TaskLocationHint.createTaskLocationHint(null, null); TaskLocationHint t7 = TaskLocationHint.createTaskLocationHint( - Sets.newHashSet(new String[] {"n1", "n2"}), Sets.newHashSet(new String[] {"r1", "r2"})); + Sets.newHashSet(new String[]{"n1", "n2"}), Sets.newHashSet(new String[]{"r1", "r2"})); TaskLocationHint t8 = TaskLocationHint.createTaskLocationHint( - Sets.newHashSet(new String[] {"n1", "n2"}), Sets.newHashSet(new String[] {"r1", "r2"})); + Sets.newHashSet(new String[]{"n1", "n2"}), Sets.newHashSet(new String[]{"r1", "r2"})); TaskLocationHint t9 = TaskLocationHint.createTaskLocationHint( - Sets.newHashSet(new String[] {"n1", "n2"}), Sets.newHashSet(new String[] {"r1"})); + Sets.newHashSet(new String[]{"n1", "n2"}), Sets.newHashSet(new String[]{"r1"})); TaskLocationHint t10 = TaskLocationHint.createTaskLocationHint( - Sets.newHashSet(new String[] {"n1"}), Sets.newHashSet(new String[] {"r1", "r2"})); + Sets.newHashSet(new String[]{"n1"}), Sets.newHashSet(new String[]{"r1", "r2"})); Assert.assertEquals(t1, t2); Assert.assertEquals(t5, t6); diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestTezConfiguration.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestTezConfiguration.java index 76d5f3c3b1..1a03a930a9 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/TestTezConfiguration.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestTezConfiguration.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -59,7 +59,7 @@ public void testKeySet() throws IllegalAccessException { for (Field f : c.getFields()) { if (!f.getName().endsWith("DEFAULT") && f.getType() == String.class && !f.getName().equals("TEZ_SITE_XML")) { - String value = (String)f.get(null); + String value = (String) f.get(null); // not prefix if (!value.endsWith(".")) { expectedKeys.add((String) f.get(null)); diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/client/TestATSHttpClient.java b/tez-api/src/test/java/org/apache/tez/dag/api/client/TestATSHttpClient.java index ef1b0a5e66..b3eaa9ca01 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/client/TestATSHttpClient.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/client/TestATSHttpClient.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -79,7 +79,7 @@ public void testGetDagStatusThrowsExceptionOnEmptyJson() throws TezException { @Test(timeout = 5000) public void testGetDagStatusSimple() throws TezException, JSONException, IOException { DAGClientTimelineImpl - httpClient = new DAGClientTimelineImpl(mock(ApplicationId.class),"EXAMPLE_DAG_ID", + httpClient = new DAGClientTimelineImpl(mock(ApplicationId.class), "EXAMPLE_DAG_ID", new TezConfiguration(), null, 0); DAGClientTimelineImpl spyClient = spy(httpClient); spyClient.baseUri = "http://yarn.ats.webapp/ws/v1/timeline"; @@ -91,9 +91,8 @@ public void testGetDagStatusSimple() throws TezException, JSONException, IOExcep Set statusOptions = new HashSet(1); statusOptions.add(StatusGetOpts.GET_COUNTERS); - final String jsonDagData = - "{ " + + "{ " + " otherinfo: { " + " status: 'SUCCEEDED'," + " diagnostics: 'SAMPLE_DIAGNOSTICS'," + @@ -108,9 +107,9 @@ public void testGetDagStatusSimple() throws TezException, JSONException, IOExcep final String jsonVertexData = "{entities:[ " + "{otherinfo: {vertexName:'v1', numTasks:5,numFailedTasks:1,numSucceededTasks:2," + - "numKilledTasks:3,numCompletedTasks:3}}," + + "numKilledTasks:3,numCompletedTasks:3}}," + "{otherinfo: {vertexName:'v2',numTasks:10,numFailedTasks:1,numSucceededTasks:5," + - "numKilledTasks:3,numCompletedTasks:4}}" + + "numKilledTasks:3,numCompletedTasks:4}}" + "]}"; doReturn(new JSONObject(jsonDagData)).when(spyClient).getJsonRootEntity(expectedDagUrl); diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java b/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java index b6d1396a2b..6f486dca2f 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -52,11 +52,10 @@ public void testPseudoAuthenticatorConnectionUrlShouldHaveUserName() throws Exce ConnectionConfigurator connConf = mock(ConnectionConfigurator.class); TimelineReaderPseudoAuthenticatedStrategy.PseudoAuthenticatedURLConnectionFactory connectionFactory = new TimelineReaderPseudoAuthenticatedStrategy - .PseudoAuthenticatedURLConnectionFactory(connConf); + .PseudoAuthenticatedURLConnectionFactory(connConf); String inputUrl = "http://host:8080/path"; String expectedUrl = inputUrl + "?user.name=" + UserGroupInformation.getCurrentUser().getShortUserName(); HttpURLConnection httpURLConnection = connectionFactory.getHttpURLConnection(new URL(inputUrl)); Assert.assertEquals(expectedUrl, httpURLConnection.getURL().toString()); } - } diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java b/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java index 0ea5d1a26d..4edbe56680 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClient.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -91,92 +91,91 @@ public class TestDAGClient { private ApplicationReport mockAppReport; private String dagIdStr; private DAGClientAMProtocolBlockingPB mockProxy; - + private VertexStatusProto vertexStatusProtoWithoutCounters; private VertexStatusProto vertexStatusProtoWithCounters; - + private DAGStatusProto dagStatusProtoWithoutCounters; private DAGStatusProto dagStatusProtoWithCounters; - - - private void setUpData(){ + + private void setUpData() { // DAG ProgressProto dagProgressProto = ProgressProto.newBuilder() - .setFailedTaskCount(1) - .setKilledTaskCount(1) - .setRunningTaskCount(2) - .setSucceededTaskCount(2) - .setTotalTaskCount(6) - .build(); - - TezCountersProto dagCountersProto=TezCountersProto.newBuilder() + .setFailedTaskCount(1) + .setKilledTaskCount(1) + .setRunningTaskCount(2) + .setSucceededTaskCount(2) + .setTotalTaskCount(6) + .build(); + + TezCountersProto dagCountersProto = TezCountersProto.newBuilder() .addCounterGroups(TezCounterGroupProto.newBuilder() .setName("DAGGroup") .addCounters(TezCounterProto.newBuilder() - .setDisplayName("dag_counter_1") - .setValue(99))) + .setDisplayName("dag_counter_1") + .setValue(99))) .build(); - + dagStatusProtoWithoutCounters = DAGStatusProto.newBuilder() - .addDiagnostics("Diagnostics_0") - .setState(DAGStatusStateProto.DAG_RUNNING) - .setDAGProgress(dagProgressProto) - .addVertexProgress( - StringProgressPairProto.newBuilder().setKey("v1") - .setProgress(ProgressProto.newBuilder() - .setFailedTaskCount(0) - .setSucceededTaskCount(0) - .setKilledTaskCount(0)) - ) - .addVertexProgress( - StringProgressPairProto.newBuilder().setKey("v2") - .setProgress(ProgressProto.newBuilder() - .setFailedTaskCount(1) - .setSucceededTaskCount(1) - .setKilledTaskCount(1)) - ) - .build(); - + .addDiagnostics("Diagnostics_0") + .setState(DAGStatusStateProto.DAG_RUNNING) + .setDAGProgress(dagProgressProto) + .addVertexProgress( + StringProgressPairProto.newBuilder().setKey("v1") + .setProgress(ProgressProto.newBuilder() + .setFailedTaskCount(0) + .setSucceededTaskCount(0) + .setKilledTaskCount(0)) + ) + .addVertexProgress( + StringProgressPairProto.newBuilder().setKey("v2") + .setProgress(ProgressProto.newBuilder() + .setFailedTaskCount(1) + .setSucceededTaskCount(1) + .setKilledTaskCount(1)) + ) + .build(); + dagStatusProtoWithCounters = DAGStatusProto.newBuilder(dagStatusProtoWithoutCounters) - .setDagCounters(dagCountersProto) - .build(); - + .setDagCounters(dagCountersProto) + .build(); + // Vertex ProgressProto vertexProgressProto = ProgressProto.newBuilder() - .setFailedTaskCount(1) - .setKilledTaskCount(0) - .setRunningTaskCount(0) - .setSucceededTaskCount(1) - .build(); - - TezCountersProto vertexCountersProto=TezCountersProto.newBuilder() - .addCounterGroups(TezCounterGroupProto.newBuilder() - .addCounters(TezCounterProto.newBuilder() - .setDisplayName("vertex_counter_1") - .setValue(99))) - .build(); - + .setFailedTaskCount(1) + .setKilledTaskCount(0) + .setRunningTaskCount(0) + .setSucceededTaskCount(1) + .build(); + + TezCountersProto vertexCountersProto = TezCountersProto.newBuilder() + .addCounterGroups(TezCounterGroupProto.newBuilder() + .addCounters(TezCounterProto.newBuilder() + .setDisplayName("vertex_counter_1") + .setValue(99))) + .build(); + vertexStatusProtoWithoutCounters = VertexStatusProto.newBuilder() - .setId("vertex_1") - .addDiagnostics("V_Diagnostics_0") - .setProgress(vertexProgressProto) - .setState(VertexStatusStateProto.VERTEX_SUCCEEDED) // make sure the waitForCompletion be able to finish - .build(); + .setId("vertex_1") + .addDiagnostics("V_Diagnostics_0") + .setProgress(vertexProgressProto) + .setState(VertexStatusStateProto.VERTEX_SUCCEEDED) // make sure the waitForCompletion be able to finish + .build(); vertexStatusProtoWithCounters = VertexStatusProto.newBuilder(vertexStatusProtoWithoutCounters) - .setVertexCounters(vertexCountersProto) - .build(); + .setVertexCounters(vertexCountersProto) + .build(); } - private static class DAGCounterRequestMatcher implements ArgumentMatcher{ + private static class DAGCounterRequestMatcher implements ArgumentMatcher { @Override public boolean matches(GetDAGStatusRequestProto requestProto) { return requestProto != null && requestProto.getStatusOptionsCount() != 0 - && requestProto.getStatusOptionsList().get(0) == StatusGetOptsProto.GET_COUNTERS; + && requestProto.getStatusOptionsList().get(0) == StatusGetOptsProto.GET_COUNTERS; } } - private static class VertexCounterRequestMatcher implements ArgumentMatcher{ + private static class VertexCounterRequestMatcher implements ArgumentMatcher { @Override public boolean matches(GetVertexStatusRequestProto requestProto) { @@ -184,12 +183,12 @@ public boolean matches(GetVertexStatusRequestProto requestProto) { && requestProto.getStatusOptionsList().get(0) == StatusGetOptsProto.GET_COUNTERS; } } - + @Before - public void setUp() throws YarnException, IOException, TezException, ServiceException{ + public void setUp() throws YarnException, IOException, TezException, ServiceException { setUpData(); - + /////////////// mock ////////////////////// mockAppId = mock(ApplicationId.class); mockAppReport = mock(ApplicationReport.class); @@ -197,55 +196,55 @@ public void setUp() throws YarnException, IOException, TezException, ServiceExce mockProxy = mock(DAGClientAMProtocolBlockingPB.class); // return the response with Counters is the request match the CounterMatcher when(mockProxy.getDAGStatus(isNull(), any())) - .thenReturn(GetDAGStatusResponseProto.newBuilder().setDagStatus(dagStatusProtoWithoutCounters).build()); + .thenReturn(GetDAGStatusResponseProto.newBuilder().setDagStatus(dagStatusProtoWithoutCounters).build()); when(mockProxy.getDAGStatus(isNull(), argThat(new DAGCounterRequestMatcher()))) - .thenReturn(GetDAGStatusResponseProto.newBuilder().setDagStatus(dagStatusProtoWithCounters).build()); - + .thenReturn(GetDAGStatusResponseProto.newBuilder().setDagStatus(dagStatusProtoWithCounters).build()); + when(mockProxy.getVertexStatus(isNull(), any())) - .thenReturn(GetVertexStatusResponseProto.newBuilder().setVertexStatus(vertexStatusProtoWithoutCounters).build()); + .thenReturn(GetVertexStatusResponseProto.newBuilder().setVertexStatus(vertexStatusProtoWithoutCounters).build()); when(mockProxy.getVertexStatus(isNull(), argThat(new VertexCounterRequestMatcher()))) - .thenReturn(GetVertexStatusResponseProto.newBuilder().setVertexStatus(vertexStatusProtoWithCounters).build()); + .thenReturn(GetVertexStatusResponseProto.newBuilder().setVertexStatus(vertexStatusProtoWithCounters).build()); TezConfiguration tezConf = new TezConfiguration(); dagClient = new DAGClientImpl(mockAppId, dagIdStr, tezConf, null, UserGroupInformation.getCurrentUser()); - DAGClientRPCImpl realClient = (DAGClientRPCImpl)((DAGClientImpl)dagClient).getRealClient(); + DAGClientRPCImpl realClient = (DAGClientRPCImpl) ((DAGClientImpl) dagClient).getRealClient(); realClient.appReport = mockAppReport; realClient.proxy = mockProxy; } - + @Test(timeout = 5000) - public void testApp() throws IOException, TezException, ServiceException{ + public void testApp() throws IOException, TezException, ServiceException { assertTrue(dagClient.getExecutionContext().contains(mockAppId.toString())); assertEquals(mockAppId.toString(), dagClient.getSessionIdentifierString()); assertEquals(dagIdStr, dagClient.getDagIdentifierString()); - DAGClientRPCImpl realClient = (DAGClientRPCImpl)((DAGClientImpl)dagClient).getRealClient(); + DAGClientRPCImpl realClient = (DAGClientRPCImpl) ((DAGClientImpl) dagClient).getRealClient(); assertEquals(mockAppReport, realClient.getApplicationReportInternal()); } - + @Test(timeout = 5000) - public void testDAGStatus() throws Exception{ + public void testDAGStatus() throws Exception { DAGStatus resultDagStatus = dagClient.getDAGStatus(null); verify(mockProxy, times(1)).getDAGStatus(null, GetDAGStatusRequestProto.newBuilder() .setDagId(dagIdStr).setTimeout(0).build()); assertEquals(new DAGStatus(dagStatusProtoWithoutCounters, DagStatusSource.AM), resultDagStatus); System.out.println("DAGStatusWithoutCounter:" + resultDagStatus); - + resultDagStatus = dagClient.getDAGStatus(Sets.newSet(StatusGetOpts.GET_COUNTERS)); verify(mockProxy, times(1)).getDAGStatus(null, GetDAGStatusRequestProto.newBuilder() .setDagId(dagIdStr).setTimeout(0).addStatusOptions(StatusGetOptsProto.GET_COUNTERS).build()); assertEquals(new DAGStatus(dagStatusProtoWithCounters, DagStatusSource.AM), resultDagStatus); System.out.println("DAGStatusWithCounter:" + resultDagStatus); } - + @Test(timeout = 5000) - public void testVertexStatus() throws Exception{ + public void testVertexStatus() throws Exception { VertexStatus resultVertexStatus = dagClient.getVertexStatus("v1", null); verify(mockProxy).getVertexStatus(null, GetVertexStatusRequestProto.newBuilder() .setDagId(dagIdStr).setVertexName("v1").build()); assertEquals(new VertexStatus(vertexStatusProtoWithoutCounters), resultVertexStatus); System.out.println("VertexWithoutCounter:" + resultVertexStatus); - + resultVertexStatus = dagClient.getVertexStatus("v1", Sets.newSet(StatusGetOpts.GET_COUNTERS)); verify(mockProxy).getVertexStatus(null, GetVertexStatusRequestProto.newBuilder() .setDagId(dagIdStr).setVertexName("v1").addStatusOptions(StatusGetOptsProto.GET_COUNTERS) @@ -253,24 +252,24 @@ public void testVertexStatus() throws Exception{ assertEquals(new VertexStatus(vertexStatusProtoWithCounters), resultVertexStatus); System.out.println("VertexWithCounter:" + resultVertexStatus); } - + @Test(timeout = 5000) - public void testTryKillDAG() throws Exception{ + public void testTryKillDAG() throws Exception { dagClient.tryKillDAG(); verify(mockProxy, times(1)).tryKillDAG(null, TryKillDAGRequestProto.newBuilder() .setDagId(dagIdStr).build()); } - + @Test(timeout = 5000) - public void testWaitForCompletion() throws Exception{ + public void testWaitForCompletion() throws Exception { // first time return DAG_RUNNING, second time return DAG_SUCCEEDED when(mockProxy.getDAGStatus(isNull(), any())) - .thenReturn(GetDAGStatusResponseProto.newBuilder().setDagStatus(dagStatusProtoWithoutCounters) - .build()) - .thenReturn(GetDAGStatusResponseProto.newBuilder().setDagStatus - (DAGStatusProto.newBuilder(dagStatusProtoWithoutCounters) - .setState(DAGStatusStateProto.DAG_SUCCEEDED).build()) - .build()); + .thenReturn(GetDAGStatusResponseProto.newBuilder().setDagStatus(dagStatusProtoWithoutCounters) + .build()) + .thenReturn(GetDAGStatusResponseProto.newBuilder().setDagStatus + (DAGStatusProto.newBuilder(dagStatusProtoWithoutCounters) + .setState(DAGStatusStateProto.DAG_SUCCEEDED).build()) + .build()); dagClient.waitForCompletion(); ArgumentCaptor rpcControllerArgumentCaptor = @@ -282,7 +281,7 @@ public void testWaitForCompletion() throws Exception{ } @Test(timeout = 5000) - public void testWaitForCompletionWithStatusUpdates() throws Exception{ + public void testWaitForCompletionWithStatusUpdates() throws Exception { // first time and second time return DAG_RUNNING, third time return DAG_SUCCEEDED when(mockProxy.getDAGStatus(isNull(), any())) @@ -296,10 +295,10 @@ public void testWaitForCompletionWithStatusUpdates() throws Exception{ DAGStatusProto.newBuilder(dagStatusProtoWithCounters) .setState(DAGStatusStateProto.DAG_RUNNING).build()).build()) .thenReturn(GetDAGStatusResponseProto.newBuilder().setDagStatus - (DAGStatusProto.newBuilder(dagStatusProtoWithoutCounters) - .setState(DAGStatusStateProto.DAG_SUCCEEDED).build()) + (DAGStatusProto.newBuilder(dagStatusProtoWithoutCounters) + .setState(DAGStatusStateProto.DAG_SUCCEEDED).build()) .build()); - + // first time for getVertexSet // second & third time for check completion ArgumentCaptor rpcControllerArgumentCaptor = @@ -322,8 +321,8 @@ public void testWaitForCompletionWithStatusUpdates() throws Exception{ DAGStatusProto.newBuilder(dagStatusProtoWithCounters) .setState(DAGStatusStateProto.DAG_RUNNING).build()).build()) .thenReturn(GetDAGStatusResponseProto.newBuilder().setDagStatus - (DAGStatusProto.newBuilder(dagStatusProtoWithCounters).setState( - DAGStatusStateProto.DAG_SUCCEEDED).build()) + (DAGStatusProto.newBuilder(dagStatusProtoWithCounters).setState( + DAGStatusStateProto.DAG_SUCCEEDED).build()) .build()); rpcControllerArgumentCaptor = @@ -388,7 +387,6 @@ public void testGetDagStatusWithTimeout() throws Exception { assertEquals(2, dagClientRpc.numGetStatusViaAmInvocations); assertEquals(DAGStatus.State.RUNNING, dagStatus.getState()); - // Fetch from AM. Success. dagClient.resetCounters(); dagClientRpc.resetCounters(); @@ -407,7 +405,6 @@ public void testGetDagStatusWithTimeout() throws Exception { // Directly from AM - previous request cached, so single invocation only. assertEquals(1, dagClientRpc.numGetStatusViaAmInvocations); assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState()); - } @Test(timeout = 5000) @@ -482,8 +479,9 @@ private static class DAGClientImplForTest extends DAGClientImpl { private DAGStatus rmDagStatus; int numGetStatusViaRmInvocations = 0; private volatile boolean faultInjected; + public DAGClientImplForTest(ApplicationId appId, String dagId, TezConfiguration conf, - @Nullable FrameworkClient frameworkClient) throws IOException { + @Nullable FrameworkClient frameworkClient) throws IOException { super(appId, dagId, conf, frameworkClient, UserGroupInformation.getCurrentUser()); } @@ -613,7 +611,7 @@ public void testGetDagStatusWithCachedStatusExpiration() throws Exception { tezConf.setLong(TezConfiguration.TEZ_DAG_STATUS_POLLINTERVAL_MS, 800L); tezConf.setLong(TezConfiguration.TEZ_CLIENT_DAG_STATUS_CACHE_TIMEOUT_SECS, 100000L); try (DAGClientImplForTest dagClientImpl = - new DAGClientImplForTest(mockAppId, dagIdStr, tezConf, null)) { + new DAGClientImplForTest(mockAppId, dagIdStr, tezConf, null)) { DAGClientRPCImplForTest dagClientRpc = new DAGClientRPCImplForTest(mockAppId, dagIdStr, tezConf, null); dagClientImpl.setRealClient(dagClientRpc); diff --git a/tez-api/src/test/java/org/apache/tez/runtime/api/event/TestCompositeDataMovementEvent.java b/tez-api/src/test/java/org/apache/tez/runtime/api/event/TestCompositeDataMovementEvent.java index 7dce6991a4..713e10152b 100644 --- a/tez-api/src/test/java/org/apache/tez/runtime/api/event/TestCompositeDataMovementEvent.java +++ b/tez-api/src/test/java/org/apache/tez/runtime/api/event/TestCompositeDataMovementEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,7 +29,7 @@ public class TestCompositeDataMovementEvent { ByteBuffer userPayload = ByteBuffer.wrap("Dummy userPayLoad".getBytes()); @Test(timeout = 5000) - public void testGetCount(){ + public void testGetCount() { int numPartitions = 2; int startIndex = 2; CompositeDataMovementEvent cdme1 = @@ -39,15 +39,14 @@ public void testGetCount(){ } @Test(timeout = 5000) - public void testGetEvents(){ + public void testGetEvents() { int numOutputs = 0; int startIndex = 1; CompositeDataMovementEvent cdme2 = CompositeDataMovementEvent.create(startIndex, numOutputs, userPayload); - for(DataMovementEvent dme: cdme2.getEvents()){ + for (DataMovementEvent dme : cdme2.getEvents()) { numOutputs++; } Assert.assertEquals(numOutputs, cdme2.getCount()); } - } diff --git a/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcher.java b/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcher.java index f9f21ca313..45fd28c18f 100644 --- a/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcher.java +++ b/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcher.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -71,9 +71,9 @@ public class AsyncDispatcher extends CompositeService implements Dispatcher { private Thread eventHandlingThread; protected final Map, EventHandler> eventHandlers = Maps.newHashMap(); protected final Map, AsyncDispatcher> eventDispatchers = Maps.newHashMap(); - protected final Map, AsyncDispatcherConcurrent> concurrentEventDispatchers = + protected final Map, AsyncDispatcherConcurrent> concurrentEventDispatchers = Maps.newHashMap(); - + private boolean exitOnDispatchException = false; public AsyncDispatcher(String name) { @@ -105,7 +105,7 @@ public void run() { Event event; try { event = eventQueue.take(); - } catch(InterruptedException ie) { + } catch (InterruptedException ie) { if (!stopped) { LOG.warn("AsyncDispatcher thread interrupted", ie); } @@ -129,7 +129,7 @@ protected void serviceStart() throws Exception { eventHandlingThread = new Thread(createThread()); eventHandlingThread.setName("Dispatcher thread {" + name + "}"); eventHandlingThread.start(); - + //start all the components super.serviceStart(); } @@ -189,9 +189,9 @@ protected void dispatch(Event event) { Class type = event.getType().getDeclaringClass(); - try{ + try { EventHandler handler = eventHandlers.get(type); - if(handler != null) { + if (handler != null) { handler.handle(event); } else { throw new Exception("No handler for registered for " + type); @@ -212,16 +212,16 @@ protected void dispatch(Event event) { } } } - + private void checkForExistingHandler(Class eventType) { EventHandler registeredHandler = (EventHandler) eventHandlers.get(eventType); - Preconditions.checkState(registeredHandler == null, + Preconditions.checkState(registeredHandler == null, "Cannot register same event on multiple dispatchers"); } private void checkForExistingDispatcher(Class eventType) { AsyncDispatcher registeredDispatcher = eventDispatchers.get(eventType); - Preconditions.checkState(registeredDispatcher == null, + Preconditions.checkState(registeredDispatcher == null, "Multiple dispatchers cannot be registered for: " + eventType.getName()); } @@ -230,7 +230,7 @@ private void checkForExistingConcurrentDispatcher(Class eventTyp Preconditions.checkState(concurrentDispatcher == null, "Multiple concurrent dispatchers cannot be registered for: " + eventType.getName()); } - + @VisibleForTesting protected void checkForExistingDispatchers(boolean checkHandler, Class eventType) { if (checkHandler) { @@ -250,7 +250,7 @@ public void enableExitOnDispatchException() { */ @Override public void register(Class eventType, - EventHandler handler) { + EventHandler handler) { Preconditions.checkState(getServiceState() == STATE.NOTINITED); /* check to see if we have a listener registered */ EventHandler registeredHandler = (EventHandler) eventHandlers.get(eventType); @@ -258,7 +258,7 @@ public void register(Class eventType, LOG.info("Registering " + eventType + " for " + handler.getClass()); if (registeredHandler == null) { eventHandlers.put(eventType, handler); - } else if (!(registeredHandler instanceof MultiListenerHandler)){ + } else if (!(registeredHandler instanceof MultiListenerHandler)) { /* for multiple listeners of an event add the multiple listener handler */ MultiListenerHandler multiHandler = new MultiListenerHandler(); multiHandler.addHandler(registeredHandler); @@ -267,36 +267,36 @@ public void register(Class eventType, } else { /* already a multilistener, just add to it */ MultiListenerHandler multiHandler - = (MultiListenerHandler) registeredHandler; + = (MultiListenerHandler) registeredHandler; multiHandler.addHandler(handler); } } - + /** * Add an EventHandler for events handled in their own dispatchers with given name */ public void registerAndCreateDispatcher(Class eventType, - EventHandler handler, String dispatcherName) { + EventHandler handler, String dispatcherName) { Preconditions.checkState(getServiceState() == STATE.NOTINITED); - + /* check to see if we have a listener registered */ checkForExistingDispatchers(true, eventType); LOG.info( - "Registering " + eventType + " for independent dispatch using: " + handler.getClass()); + "Registering " + eventType + " for independent dispatch using: " + handler.getClass()); AsyncDispatcher dispatcher = new AsyncDispatcher(dispatcherName); dispatcher.register(eventType, handler); eventDispatchers.put(eventType, dispatcher); addIfService(dispatcher); } - + public AsyncDispatcherConcurrent registerAndCreateDispatcher(Class eventType, - EventHandler handler, String dispatcherName, int numThreads) { + EventHandler handler, String dispatcherName, int numThreads) { Preconditions.checkState(getServiceState() == STATE.NOTINITED); - + /* check to see if we have a listener registered */ checkForExistingDispatchers(true, eventType); LOG.info( - "Registering " + eventType + " for concurrent dispatch using: " + handler.getClass()); + "Registering " + eventType + " for concurrent dispatch using: " + handler.getClass()); AsyncDispatcherConcurrent dispatcher = new AsyncDispatcherConcurrent(dispatcherName, numThreads); if (exitOnDispatchException) { dispatcher.enableExitOnDispatchException(); @@ -306,15 +306,15 @@ public AsyncDispatcherConcurrent registerAndCreateDispatcher(Class eventType, - EventHandler handler, AsyncDispatcherConcurrent dispatcher) { + EventHandler handler, AsyncDispatcherConcurrent dispatcher) { Preconditions.checkState(getServiceState() == STATE.NOTINITED); - + /* check to see if we have a listener registered */ checkForExistingDispatchers(true, eventType); LOG.info("Registering " + eventType + " with existing concurrent dispatch using: " - + handler.getClass()); + + handler.getClass()); if (exitOnDispatchException) { dispatcher.enableExitOnDispatchException(); } @@ -349,12 +349,12 @@ public void handle(Event event) { concurrentDispatcher.getEventHandler().handle(event); return; } - + // no registered dispatcher. use internal dispatcher. - + /* all this method does is enqueue all the events onto the queue */ int qSize = eventQueue.size(); - if (qSize !=0 && qSize %1000 == 0) { + if (qSize != 0 && qSize % 1000 == 0) { LOG.info("Size of event-queue is " + qSize); } int remCapacity = eventQueue.remainingCapacity(); @@ -370,7 +370,9 @@ public void handle(Event event) { } throw new YarnRuntimeException(e); } - }; + } + + ; } /** @@ -387,7 +389,7 @@ public MultiListenerHandler() { @Override public void handle(Event event) { - for (EventHandler handler: listofHandlers) { + for (EventHandler handler : listofHandlers) { handler.handle(event); } } @@ -395,7 +397,6 @@ public void handle(Event event) { void addHandler(EventHandler handler) { listofHandlers.add(handler); } - } Runnable createShutDownThread() { diff --git a/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcherConcurrent.java b/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcherConcurrent.java index f22159b46b..b761cdcdc0 100644 --- a/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcherConcurrent.java +++ b/tez-common/src/main/java/org/apache/tez/common/AsyncDispatcherConcurrent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -73,9 +73,9 @@ public class AsyncDispatcherConcurrent extends CompositeService implements Dispa private ExecutorService execService; private final int numThreads; - + protected final Map, EventHandler> eventHandlers = Maps.newHashMap(); - protected final Map, AsyncDispatcherConcurrent> eventDispatchers = + protected final Map, AsyncDispatcherConcurrent> eventDispatchers = Maps.newHashMap(); private boolean exitOnDispatchException = false; @@ -86,14 +86,14 @@ public class AsyncDispatcherConcurrent extends CompositeService implements Dispa this.eventQueues = Lists.newArrayListWithCapacity(numThreads); this.numThreads = numThreads; } - + class DispatchRunner implements Runnable { final LinkedBlockingQueue queue; - + public DispatchRunner(LinkedBlockingQueue queue) { this.queue = queue; } - + @Override public void run() { while (!stopped && !Thread.currentThread().isInterrupted()) { @@ -111,7 +111,7 @@ public void run() { Event event; try { event = queue.take(); - } catch(InterruptedException ie) { + } catch (InterruptedException ie) { if (!stopped) { LOG.warn("AsyncDispatcher thread interrupted", ie); } @@ -122,8 +122,10 @@ public void run() { } } } - }; - + } + + ; + @Override protected void serviceInit(Configuration conf) throws Exception { super.serviceInit(conf); @@ -133,10 +135,10 @@ protected void serviceInit(Configuration conf) throws Exception { protected void serviceStart() throws Exception { execService = Executors.newFixedThreadPool(numThreads, new ThreadFactoryBuilder().setDaemon(true) .setNameFormat("Dispatcher {" + this.name + "} #%d").build()); - for (int i=0; i()); } - for (int i=0; i type = event.getType().getDeclaringClass(); - try{ + try { EventHandler handler = eventHandlers.get(type); - if(handler != null) { + if (handler != null) { handler.handle(event); } else { throw new Exception("No handler for registered for " + type); @@ -205,7 +207,7 @@ protected void dispatch(Event event) { private void checkForExistingHandler(Class eventType) { EventHandler registeredHandler = (EventHandler) eventHandlers.get(eventType); - Preconditions.checkState(registeredHandler == null, + Preconditions.checkState(registeredHandler == null, "Cannot register same event on multiple dispatchers"); } @@ -227,7 +229,7 @@ private void checkForExistingDispatchers(boolean checkHandler, Class eventType, - EventHandler handler) { + EventHandler handler) { Preconditions.checkState(getServiceState() == STATE.NOTINITED); /* check to see if we have a listener registered */ EventHandler registeredHandler = (EventHandler) eventHandlers.get(eventType); @@ -235,7 +237,7 @@ public void register(Class eventType, LOG.info("Registering " + eventType + " for " + handler.getClass()); if (registeredHandler == null) { eventHandlers.put(eventType, handler); - } else if (!(registeredHandler instanceof MultiListenerHandler)){ + } else if (!(registeredHandler instanceof MultiListenerHandler)) { /* for multiple listeners of an event add the multiple listener handler */ MultiListenerHandler multiHandler = new MultiListenerHandler(); multiHandler.addHandler(registeredHandler); @@ -244,38 +246,38 @@ public void register(Class eventType, } else { /* already a multilistener, just add to it */ MultiListenerHandler multiHandler - = (MultiListenerHandler) registeredHandler; + = (MultiListenerHandler) registeredHandler; multiHandler.addHandler(handler); } } - + /** * Add an EventHandler for events handled in their own dispatchers with given name and threads */ - + public AsyncDispatcherConcurrent registerAndCreateDispatcher(Class eventType, - EventHandler handler, String dispatcherName, int numThreads) { + EventHandler handler, String dispatcherName, int numThreads) { Preconditions.checkState(getServiceState() == STATE.NOTINITED); - + /* check to see if we have a listener registered */ checkForExistingDispatchers(true, eventType); LOG.info( - "Registering " + eventType + " for independent dispatch using: " + handler.getClass()); + "Registering " + eventType + " for independent dispatch using: " + handler.getClass()); AsyncDispatcherConcurrent dispatcher = new AsyncDispatcherConcurrent(dispatcherName, numThreads); dispatcher.register(eventType, handler); eventDispatchers.put(eventType, dispatcher); addIfService(dispatcher); return dispatcher; } - + public void registerWithExistingDispatcher(Class eventType, - EventHandler handler, AsyncDispatcherConcurrent dispatcher) { + EventHandler handler, AsyncDispatcherConcurrent dispatcher) { Preconditions.checkState(getServiceState() == STATE.NOTINITED); - + /* check to see if we have a listener registered */ checkForExistingDispatchers(true, eventType); LOG.info("Registering " + eventType + " wit existing concurrent dispatch using: " - + handler.getClass()); + + handler.getClass()); dispatcher.register(eventType, handler); eventDispatchers.put(eventType, dispatcher); } @@ -299,7 +301,7 @@ public void handle(TezAbstractEvent event) { return; } drained = false; - + // offload to specific dispatcher if one exists Class type = event.getType().getDeclaringClass(); AsyncDispatcherConcurrent registeredDispatcher = eventDispatchers.get(type); @@ -307,14 +309,14 @@ public void handle(TezAbstractEvent event) { registeredDispatcher.getEventHandler().handle(event); return; } - + int index = numThreads > 1 ? event.getSerializingHash() % numThreads : 0; - // no registered dispatcher. use internal dispatcher. + // no registered dispatcher. use internal dispatcher. LinkedBlockingQueue queue = eventQueues.get(index); /* all this method does is enqueue all the events onto the queue */ int qSize = queue.size(); - if (qSize !=0 && qSize %1000 == 0) { + if (qSize != 0 && qSize % 1000 == 0) { LOG.info("Size of event-queue is " + qSize); } int remCapacity = queue.remainingCapacity(); @@ -330,7 +332,9 @@ public void handle(TezAbstractEvent event) { } throw new YarnRuntimeException(e); } - }; + } + + ; } /** @@ -347,7 +351,7 @@ public MultiListenerHandler() { @Override public void handle(Event event) { - for (EventHandler handler: listofHandlers) { + for (EventHandler handler : listofHandlers) { handler.handle(event); } } @@ -355,7 +359,6 @@ public void handle(Event event) { void addHandler(EventHandler handler) { listofHandlers.add(handler); } - } Runnable createShutDownThread() { diff --git a/tez-common/src/main/java/org/apache/tez/common/CallableWithNdc.java b/tez-common/src/main/java/org/apache/tez/common/CallableWithNdc.java index 2d5e1dde21..7d8394fb57 100644 --- a/tez-common/src/main/java/org/apache/tez/common/CallableWithNdc.java +++ b/tez-common/src/main/java/org/apache/tez/common/CallableWithNdc.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-common/src/main/java/org/apache/tez/common/DagContainerLauncher.java b/tez-common/src/main/java/org/apache/tez/common/DagContainerLauncher.java index c2337af77e..b1f11d0580 100644 --- a/tez-common/src/main/java/org/apache/tez/common/DagContainerLauncher.java +++ b/tez-common/src/main/java/org/apache/tez/common/DagContainerLauncher.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-common/src/main/java/org/apache/tez/common/GcTimeUpdater.java b/tez-common/src/main/java/org/apache/tez/common/GcTimeUpdater.java index 344c08aabf..db21ac3100 100644 --- a/tez-common/src/main/java/org/apache/tez/common/GcTimeUpdater.java +++ b/tez-common/src/main/java/org/apache/tez/common/GcTimeUpdater.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -40,7 +40,7 @@ public GcTimeUpdater(TezCounters counters) { getElapsedGc(); // Initialize 'lastGcMillis' with the current time spent. this.counters = counters; } - + public long getCumulativaGcTime() { long thisGcMillis = 0; for (GarbageCollectorMXBean gcBean : gcBeans) { diff --git a/tez-common/src/main/java/org/apache/tez/common/Preconditions.java b/tez-common/src/main/java/org/apache/tez/common/Preconditions.java index 42d862cd22..714066cd2f 100644 --- a/tez-common/src/main/java/org/apache/tez/common/Preconditions.java +++ b/tez-common/src/main/java/org/apache/tez/common/Preconditions.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -40,7 +40,7 @@ public static void checkArgument(boolean expression, @Nullable Object message) { } public static void checkArgument(boolean expression, @Nullable String template, - @Nullable Object... args) { + @Nullable Object... args) { if (!expression) { throw new IllegalArgumentException(format(template, args)); } @@ -59,7 +59,7 @@ public static void checkState(boolean expression, @Nullable Object message) { } public static void checkState(boolean expression, @Nullable String template, - @Nullable Object... args) { + @Nullable Object... args) { if (!expression) { throw new IllegalStateException(format(template, args)); } @@ -69,7 +69,7 @@ private static String format(@Nullable String template, @Nullable Object... args template = String.valueOf(template); // null -> "null" if (args == null) { - args = new Object[] { "(Object[])null" }; + args = new Object[]{"(Object[])null"}; } else { for (int i = 0; i < args.length; i++) { args[i] = lenientToString(args[i]); diff --git a/tez-common/src/main/java/org/apache/tez/common/RunnableWithNdc.java b/tez-common/src/main/java/org/apache/tez/common/RunnableWithNdc.java index 4d718d46cf..f3cf2d962b 100644 --- a/tez-common/src/main/java/org/apache/tez/common/RunnableWithNdc.java +++ b/tez-common/src/main/java/org/apache/tez/common/RunnableWithNdc.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-common/src/main/java/org/apache/tez/common/StreamHelper.java b/tez-common/src/main/java/org/apache/tez/common/StreamHelper.java index 789d9b4268..1c89178489 100644 --- a/tez-common/src/main/java/org/apache/tez/common/StreamHelper.java +++ b/tez-common/src/main/java/org/apache/tez/common/StreamHelper.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-common/src/main/java/org/apache/tez/common/TezAbstractEvent.java b/tez-common/src/main/java/org/apache/tez/common/TezAbstractEvent.java index b73611202f..949b01be70 100644 --- a/tez-common/src/main/java/org/apache/tez/common/TezAbstractEvent.java +++ b/tez-common/src/main/java/org/apache/tez/common/TezAbstractEvent.java @@ -1,27 +1,26 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.common; /** * Event that allows running in parallel for different instances - * + * * @param * Event type */ @@ -35,7 +34,7 @@ public TezAbstractEvent(TYPE type) { /** * Returning a number that is identical for event instances that need to be * serialized while processing. - * + * * @return Serializing identifier. Not overriding this causes serialization * for all events instances */ diff --git a/tez-common/src/main/java/org/apache/tez/common/TezContainerLogAppender.java b/tez-common/src/main/java/org/apache/tez/common/TezContainerLogAppender.java index 1649a0b318..925e72b531 100644 --- a/tez-common/src/main/java/org/apache/tez/common/TezContainerLogAppender.java +++ b/tez-common/src/main/java/org/apache/tez/common/TezContainerLogAppender.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -27,7 +27,7 @@ /** * A simple log4j-appender for a tez container's logs. - * + * */ @Unstable public class TezContainerLogAppender extends FileAppender { @@ -47,7 +47,7 @@ public void activateOptions() { /** * Set the name of the file for logging. This should NOT be an absolute path. * The file will be created within the container's log directory. - * + * * @param fileName * @throws NullPointerException if {@code fileName} is {@code null} * @throws IllegalArgumentException if {@code fileName} is an absolute path diff --git a/tez-common/src/main/java/org/apache/tez/common/TezExecutors.java b/tez-common/src/main/java/org/apache/tez/common/TezExecutors.java index a74c8ad22f..fa774b5419 100644 --- a/tez-common/src/main/java/org/apache/tez/common/TezExecutors.java +++ b/tez-common/src/main/java/org/apache/tez/common/TezExecutors.java @@ -33,7 +33,7 @@ public interface TezExecutors { /** * Create a ExecutorService with the given parameters. * - * @param parallelism Represents total number of tasks to be executed in parallel. + * @param parallelism Represents total number of tasks to be executed in parallel. * @param threadNameFormat The name the thread should take when executing tasks from this executor * @return An ExecutorService. */ diff --git a/tez-common/src/main/java/org/apache/tez/common/TezLog4jConfigurator.java b/tez-common/src/main/java/org/apache/tez/common/TezLog4jConfigurator.java index 03324f7a2b..7cb246c5d8 100644 --- a/tez-common/src/main/java/org/apache/tez/common/TezLog4jConfigurator.java +++ b/tez-common/src/main/java/org/apache/tez/common/TezLog4jConfigurator.java @@ -21,7 +21,6 @@ import java.util.Locale; import java.util.Properties; - import org.apache.log4j.PropertyConfigurator; import org.apache.log4j.spi.LoggerRepository; import org.apache.tez.dag.api.TezConstants; @@ -31,9 +30,9 @@ public class TezLog4jConfigurator extends PropertyConfigurator { public void doConfigure(Properties properties, LoggerRepository repository) { String logParams = System.getenv(TezConstants.TEZ_CONTAINER_LOG_PARAMS); if (logParams != null) { - String []parts = logParams.split(TezConstants.TEZ_CONTAINER_LOG_PARAMS_SEPARATOR); + String[] parts = logParams.split(TezConstants.TEZ_CONTAINER_LOG_PARAMS_SEPARATOR); for (String logParam : parts) { - String [] logParamParts = logParam.split("="); + String[] logParamParts = logParam.split("="); if (logParamParts.length == 2) { String loggerName = "log4j.logger." + logParamParts[0]; String logLevel = logParamParts[1].toUpperCase(Locale.ENGLISH); @@ -47,5 +46,4 @@ public void doConfigure(Properties properties, LoggerRepository repository) { } super.doConfigure(properties, repository); } - } diff --git a/tez-common/src/main/java/org/apache/tez/common/TezSharedExecutor.java b/tez-common/src/main/java/org/apache/tez/common/TezSharedExecutor.java index bf8eb4f861..2826bb0b69 100644 --- a/tez-common/src/main/java/org/apache/tez/common/TezSharedExecutor.java +++ b/tez-common/src/main/java/org/apache/tez/common/TezSharedExecutor.java @@ -242,7 +242,7 @@ private void removeFuture(ManagedFutureTask future) { @Override protected ManagedFutureTask newTaskFor(Runnable runnable, T value) { if (runnable instanceof ManagedFutureTask) { - return (ManagedFutureTask)runnable; + return (ManagedFutureTask) runnable; } return new ManagedFutureTask(runnable, value); } diff --git a/tez-common/src/main/java/org/apache/tez/common/TezTestUtils.java b/tez-common/src/main/java/org/apache/tez/common/TezTestUtils.java index 4b9d5c3218..ff82dd8fc4 100644 --- a/tez-common/src/main/java/org/apache/tez/common/TezTestUtils.java +++ b/tez-common/src/main/java/org/apache/tez/common/TezTestUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,7 +31,7 @@ public class TezTestUtils { public static void ensureHighDiskUtilizationLimit(Configuration conf) { if (conf.getFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, YarnConfiguration.DEFAULT_NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE) == - YarnConfiguration.DEFAULT_NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE) { + YarnConfiguration.DEFAULT_NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE) { conf.setFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 99.0f); } } diff --git a/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java b/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java index adcae8a964..6b10b57546 100644 --- a/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java +++ b/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -227,7 +227,7 @@ public static String convertDagPlanToString(DAGProtos.DAGPlan dagPlan) throws IO Credentials credentials = DagTypeConverters.convertByteStringToCredentials(dagPlan.getCredentialsBinary()); TextFormat.printField(entry.getKey(), - ByteString.copyFrom(TezCommonUtils.getCredentialsInfo(credentials,"dag").getBytes( + ByteString.copyFrom(TezCommonUtils.getCredentialsInfo(credentials, "dag").getBytes( Charset.forName("UTF-8"))), sb); } } @@ -307,7 +307,7 @@ public static TaskAttemptEndReason toTaskAttemptEndReason(TaskAttemptTermination } public static > Set getEnums(Configuration conf, String confName, - Class enumType, String defaultValues) { + Class enumType, String defaultValues) { String[] names = conf.getStrings(confName); if (names == null) { names = StringUtils.getStrings(defaultValues); @@ -354,7 +354,7 @@ public static void setSecurityUtilConfigration(Logger log, Configuration conf) { } catch (NoSuchMethodException e) { // This is not available, so ignore it. } catch (SecurityException | IllegalAccessException | IllegalArgumentException | - InvocationTargetException e) { + InvocationTargetException e) { log.warn("Error invoking SecurityUtil.setConfiguration: ", e); throw new TezUncheckedException("Error invoking SecurityUtil.setConfiguration", e); } diff --git a/tez-common/src/main/java/org/apache/tez/common/io/NonSyncByteArrayInputStream.java b/tez-common/src/main/java/org/apache/tez/common/io/NonSyncByteArrayInputStream.java index 5b6e52ae6c..c996045b12 100644 --- a/tez-common/src/main/java/org/apache/tez/common/io/NonSyncByteArrayInputStream.java +++ b/tez-common/src/main/java/org/apache/tez/common/io/NonSyncByteArrayInputStream.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-common/src/main/java/org/apache/tez/common/io/NonSyncByteArrayOutputStream.java b/tez-common/src/main/java/org/apache/tez/common/io/NonSyncByteArrayOutputStream.java index 40fae6fd7a..1882c0d70d 100644 --- a/tez-common/src/main/java/org/apache/tez/common/io/NonSyncByteArrayOutputStream.java +++ b/tez-common/src/main/java/org/apache/tez/common/io/NonSyncByteArrayOutputStream.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -101,13 +101,15 @@ public void writeTo(OutputStream out) throws IOException { /** * {@inheritDoc} */ - @Override public byte toByteArray()[] { + @Override + public byte toByteArray()[] { return Arrays.copyOf(buf, count); } /** * {@inheritDoc} - */ public int size() { + */ + public int size() { return count; } } diff --git a/tez-common/src/main/java/org/apache/tez/common/io/NonSyncDataOutputStream.java b/tez-common/src/main/java/org/apache/tez/common/io/NonSyncDataOutputStream.java index d6302fe9a9..8bb8d58515 100644 --- a/tez-common/src/main/java/org/apache/tez/common/io/NonSyncDataOutputStream.java +++ b/tez-common/src/main/java/org/apache/tez/common/io/NonSyncDataOutputStream.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-common/src/main/java/org/apache/tez/common/package-info.java b/tez-common/src/main/java/org/apache/tez/common/package-info.java index 88a5e8607a..442badf037 100644 --- a/tez-common/src/main/java/org/apache/tez/common/package-info.java +++ b/tez-common/src/main/java/org/apache/tez/common/package-info.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,4 +19,4 @@ @Private package org.apache.tez.common; -import org.apache.hadoop.classification.InterfaceAudience.Private; \ No newline at end of file +import org.apache.hadoop.classification.InterfaceAudience.Private; diff --git a/tez-common/src/main/java/org/apache/tez/common/web/AbstractServletToControllerAdapter.java b/tez-common/src/main/java/org/apache/tez/common/web/AbstractServletToControllerAdapter.java index b79b5d5d9c..8d5dadb921 100644 --- a/tez-common/src/main/java/org/apache/tez/common/web/AbstractServletToControllerAdapter.java +++ b/tez-common/src/main/java/org/apache/tez/common/web/AbstractServletToControllerAdapter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -58,8 +58,9 @@ public void index() { this.servlet.getClass().getMethod("doGet", HttpServletRequest.class, HttpServletResponse.class); doGetMethod.setAccessible(true); doGetMethod.invoke(this.servlet, request(), response()); - } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException - | SecurityException e) { + } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException | + NoSuchMethodException + | SecurityException e) { throw new RuntimeException(e); } } diff --git a/tez-common/src/main/java/org/apache/tez/common/web/ServletToControllerAdapters.java b/tez-common/src/main/java/org/apache/tez/common/web/ServletToControllerAdapters.java index 35ca1b6408..230fd43459 100644 --- a/tez-common/src/main/java/org/apache/tez/common/web/ServletToControllerAdapters.java +++ b/tez-common/src/main/java/org/apache/tez/common/web/ServletToControllerAdapters.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-common/src/main/java/org/apache/tez/common/web/package-info.java b/tez-common/src/main/java/org/apache/tez/common/web/package-info.java index 2fbda31fda..3a2d6812ac 100644 --- a/tez-common/src/main/java/org/apache/tez/common/web/package-info.java +++ b/tez-common/src/main/java/org/apache/tez/common/web/package-info.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,4 +19,4 @@ @Private package org.apache.tez.common.web; -import org.apache.hadoop.classification.InterfaceAudience.Private; \ No newline at end of file +import org.apache.hadoop.classification.InterfaceAudience.Private; diff --git a/tez-common/src/main/java/org/apache/tez/dag/history/logging/EntityTypes.java b/tez-common/src/main/java/org/apache/tez/dag/history/logging/EntityTypes.java index 6f6205d398..f85ea56142 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/history/logging/EntityTypes.java +++ b/tez-common/src/main/java/org/apache/tez/dag/history/logging/EntityTypes.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/DAGIDAware.java b/tez-common/src/main/java/org/apache/tez/dag/records/DAGIDAware.java index 1234a30053..b7e95124c0 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/records/DAGIDAware.java +++ b/tez-common/src/main/java/org/apache/tez/dag/records/DAGIDAware.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,9 +21,9 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; public interface DAGIDAware { - TezDAGID getDAGID(); + TezDAGID getDAGID(); - default ApplicationId getApplicationId() { - return getDAGID().getApplicationId(); - } + default ApplicationId getApplicationId() { + return getDAGID().getApplicationId(); + } } diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/DagIdentifierImpl.java b/tez-common/src/main/java/org/apache/tez/dag/records/DagIdentifierImpl.java index 099cb58a13..9fad29fb18 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/records/DagIdentifierImpl.java +++ b/tez-common/src/main/java/org/apache/tez/dag/records/DagIdentifierImpl.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,7 +24,7 @@ public class DagIdentifierImpl implements DagIdentifier { private final TezDAGID dagId; private final String dagName; - + public DagIdentifierImpl(String dagName, TezDAGID dagId) { this.dagId = dagId; this.dagName = dagName; @@ -39,29 +39,28 @@ public String getName() { public int getIdentifier() { return dagId.getId(); } - + @Override public boolean equals(Object o) { if (this == o) { return true; } - if(o == null) { + if (o == null) { return false; } if (o.getClass() == this.getClass()) { DagIdentifierImpl other = (DagIdentifierImpl) o; return this.dagId.equals(other.dagId); - } - else { + } else { return false; } } - + @Override public String toString() { return "Dag: " + dagName + ":[" + getIdentifier() + "]"; } - + @Override public int hashCode() { return dagId.hashCode(); diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TaskAttemptIDAware.java b/tez-common/src/main/java/org/apache/tez/dag/records/TaskAttemptIDAware.java index 924fd07109..30026060ac 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/records/TaskAttemptIDAware.java +++ b/tez-common/src/main/java/org/apache/tez/dag/records/TaskAttemptIDAware.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,10 +19,10 @@ package org.apache.tez.dag.records; public interface TaskAttemptIDAware extends TaskIDAware { - TezTaskAttemptID getTaskAttemptID(); + TezTaskAttemptID getTaskAttemptID(); - @Override - default TezTaskID getTaskID() { - return getTaskAttemptID().getTaskID(); - } + @Override + default TezTaskID getTaskID() { + return getTaskAttemptID().getTaskID(); + } } diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TaskAttemptIdentifierImpl.java b/tez-common/src/main/java/org/apache/tez/dag/records/TaskAttemptIdentifierImpl.java index b834111cb9..887ea5f68c 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/records/TaskAttemptIdentifierImpl.java +++ b/tez-common/src/main/java/org/apache/tez/dag/records/TaskAttemptIdentifierImpl.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,7 +25,7 @@ public class TaskAttemptIdentifierImpl implements TaskAttemptIdentifier { private final TaskIdentifier taskIdentifier; private final TezTaskAttemptID attemptId; - + public TaskAttemptIdentifierImpl(String dagName, String vertexName, TezTaskAttemptID attemptId) { this.attemptId = attemptId; this.taskIdentifier = new TaskIdentifierImpl(dagName, vertexName, attemptId.getTaskID()); @@ -35,34 +35,33 @@ public TaskAttemptIdentifierImpl(String dagName, String vertexName, TezTaskAttem public int getIdentifier() { return attemptId.getId(); } - + @Override public TaskIdentifier getTaskIdentifier() { return taskIdentifier; } - + @Override public boolean equals(Object o) { if (this == o) { return true; } - if(o == null) { + if (o == null) { return false; } if (o.getClass() == this.getClass()) { TaskAttemptIdentifierImpl other = (TaskAttemptIdentifierImpl) o; return this.attemptId.equals(other.attemptId); - } - else { + } else { return false; } } - + @Override public String toString() { return taskIdentifier.toString() + " Attempt: [" + getIdentifier() + "]"; } - + @Override public int hashCode() { return attemptId.hashCode(); diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TaskAttemptTerminationCause.java b/tez-common/src/main/java/org/apache/tez/dag/records/TaskAttemptTerminationCause.java index 14eaa3a8db..3003847154 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/records/TaskAttemptTerminationCause.java +++ b/tez-common/src/main/java/org/apache/tez/dag/records/TaskAttemptTerminationCause.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,7 +20,7 @@ public enum TaskAttemptTerminationCause { UNKNOWN_ERROR, // The error cause is unknown. Usually means a gap in error propagation - + TERMINATED_BY_CLIENT, // Killed by client command TERMINATED_AT_SHUTDOWN, // Killed due execution shutdown TERMINATED_AT_RECOVERY, // Killed in recovery, due to can not recover running task attempt @@ -29,7 +29,7 @@ public enum TaskAttemptTerminationCause { TERMINATED_INEFFECTIVE_SPECULATION, // Killed speculative attempt because original succeeded TERMINATED_EFFECTIVE_SPECULATION, // Killed original attempt because speculation succeeded TERMINATED_ORPHANED, // Attempt is no longer needed by the task - + APPLICATION_ERROR, // Failed due to application code error FRAMEWORK_ERROR, // Failed due to code error in Tez code INPUT_READ_ERROR, // Failed due to error in reading inputs @@ -37,7 +37,7 @@ public enum TaskAttemptTerminationCause { OUTPUT_LOST, // Failed because attempts output were reported lost NO_PROGRESS, // Failed because no progress was being made TASK_HEARTBEAT_ERROR, // Failed because AM lost connection to the task - + CONTAINER_LAUNCH_FAILED, // Failed to launch container CONTAINER_EXITED, // Container exited. Indicates gap in specific error propagation from the cluster CONTAINER_STOPPED, // Container stopped or released by Tez @@ -48,5 +48,4 @@ public enum TaskAttemptTerminationCause { SERVICE_BUSY, // Service rejected the task INTERRUPTED_BY_SYSTEM, // Interrupted by the system. e.g. Pre-emption INTERRUPTED_BY_USER, // Interrupted by the user - } diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TaskIDAware.java b/tez-common/src/main/java/org/apache/tez/dag/records/TaskIDAware.java index 0bee45dfa1..776cb32228 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/records/TaskIDAware.java +++ b/tez-common/src/main/java/org/apache/tez/dag/records/TaskIDAware.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,10 +19,10 @@ package org.apache.tez.dag.records; public interface TaskIDAware extends VertexIDAware { - TezTaskID getTaskID(); + TezTaskID getTaskID(); - @Override - default TezVertexID getVertexID() { - return getTaskID().getVertexID(); - } + @Override + default TezVertexID getVertexID() { + return getTaskID().getVertexID(); + } } diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TaskIdentifierImpl.java b/tez-common/src/main/java/org/apache/tez/dag/records/TaskIdentifierImpl.java index fb0848aaad..004de8d5d8 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/records/TaskIdentifierImpl.java +++ b/tez-common/src/main/java/org/apache/tez/dag/records/TaskIdentifierImpl.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,7 +25,7 @@ public class TaskIdentifierImpl implements TaskIdentifier { private final VertexIdentifier vertexIdentifier; private final TezTaskID taskId; - + public TaskIdentifierImpl(String dagName, String vertexName, TezTaskID taskId) { this.taskId = taskId; this.vertexIdentifier = new VertexIdentifierImpl(dagName, vertexName, taskId.getVertexID()); @@ -35,29 +35,28 @@ public TaskIdentifierImpl(String dagName, String vertexName, TezTaskID taskId) { public int getIdentifier() { return taskId.getId(); } - + @Override public VertexIdentifier getVertexIdentifier() { return vertexIdentifier; } - + @Override public boolean equals(Object o) { if (this == o) { return true; } - if(o == null) { + if (o == null) { return false; } if (o.getClass() == this.getClass()) { TaskIdentifierImpl other = (TaskIdentifierImpl) o; return this.taskId.equals(other.taskId); - } - else { + } else { return false; } } - + @Override public String toString() { return vertexIdentifier.toString() + " Task [" + getIdentifier() + "]"; diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java index c46aa6088b..4789888010 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java +++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezDAGID.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -58,7 +58,7 @@ public static TezDAGID getInstance(ApplicationId applicationId, int id) { Objects.requireNonNull(applicationId, "ApplicationID cannot be null"); return tezDAGIDCache.intern(new TezDAGID(applicationId, id)); } - + /** * Get a DAGID object from given parts. * @param yarnRMIdentifier YARN RM identifier @@ -83,7 +83,6 @@ private TezDAGID(ApplicationId applicationId, int id) { this.applicationId = applicationId; } - private TezDAGID(String yarnRMIdentifier, int appId, int id) { this(ApplicationId.newInstance(Long.parseLong(yarnRMIdentifier), appId), id); @@ -99,18 +98,17 @@ public boolean equals(Object o) { if (!super.equals(o)) return false; - TezDAGID that = (TezDAGID)o; + TezDAGID that = (TezDAGID) o; return this.applicationId.equals(that.applicationId); } /**Compare TaskInProgressIds by first jobIds, then by tip numbers and type.*/ @Override public int compareTo(TezID o) { - TezDAGID that = (TezDAGID)o; + TezDAGID that = (TezDAGID) o; return this.applicationId.compareTo(that.applicationId); } - @Override // Can't do much about this instance if used via the RPC layer. Any downstream // users can however avoid using this method. @@ -127,7 +125,7 @@ public static TezDAGID readTezDAGID(DataInput in) throws IOException { TezDAGID dagID = getInstance(ApplicationId.newInstance(clusterId, appId), dagIdInt); return dagID; } - + @Override public void write(DataOutput out) throws IOException { out.writeLong(applicationId.getClusterTimestamp()); @@ -216,5 +214,4 @@ protected StringBuilder appendTo(StringBuilder builder) { public int hashCode() { return applicationId.hashCode() * 524287 + id; } - } diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezID.java index 7efbd9a889..0690875d53 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/records/TezID.java +++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezID.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -30,7 +30,7 @@ * A general identifier, which internally stores the id * as an integer. This is the super class of {@link TezDAGID}, * {@link TezVertexID}, {@link TezTaskID}, and {@link TezTaskAttemptID}. - * + * * @see TezTaskID * @see TezTaskAttemptID */ @@ -67,13 +67,12 @@ public int hashCode() { public boolean equals(Object o) { if (this == o) return true; - if(o == null) + if (o == null) return false; if (o.getClass() == this.getClass()) { TezID that = (TezID) o; return this.id == that.id; - } - else + } else return false; } @@ -86,7 +85,7 @@ public int compareTo(TezID that) { public void readFields(DataInput in) throws IOException { this.id = in.readInt(); } - + public static int readID(DataInput in) throws IOException { return in.readInt(); } @@ -95,5 +94,4 @@ public static int readID(DataInput in) throws IOException { public void write(DataOutput out) throws IOException { out.writeInt(id); } - } diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskAttemptID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskAttemptID.java index fe2b84449f..f817359880 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskAttemptID.java +++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskAttemptID.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -55,12 +55,12 @@ public class TezTaskAttemptID extends TezID implements TaskIDAware { // Public for Writable serialization. Verify if this is actually required. public TezTaskAttemptID() { } - + /** * Constructs a TaskAttemptID object from given {@link TezTaskID}. * @param taskID TaskID that this task belongs to * @param id the task attempt number - * @throws NullPointerException if {@code taskID} is {@code null} + * @throws NullPointerException if {@code taskID} is {@code null} */ public static TezTaskAttemptID getInstance(TezTaskID taskID, int id) { Objects.requireNonNull(taskID); @@ -77,16 +77,16 @@ private TezTaskAttemptID(TezTaskID taskId, int id) { public TezTaskID getTaskID() { return taskId; } - + @Override public boolean equals(Object o) { if (!super.equals(o)) return false; - TezTaskAttemptID that = (TezTaskAttemptID)o; + TezTaskAttemptID that = (TezTaskAttemptID) o; return this.taskId.equals(that.taskId); } - + /** * Add the unique string to the StringBuilder * @param builder the builder to append ot @@ -97,27 +97,27 @@ protected StringBuilder appendTo(StringBuilder builder) { builder.append(SEPARATOR); return builder.append(id); } - + @Override public int hashCode() { return taskId.hashCode() * 539501 + id; } - + /**Compare TaskIds by first tipIds, then by task numbers. */ @Override public int compareTo(TezID o) { - TezTaskAttemptID that = (TezTaskAttemptID)o; + TezTaskAttemptID that = (TezTaskAttemptID) o; int tipComp = this.taskId.compareTo(that.taskId); - if(tipComp == 0) { + if (tipComp == 0) { return this.id - that.id; - } - else return tipComp; + } else return tipComp; } + @Override - public String toString() { + public String toString() { return appendTo(new StringBuilder(ATTEMPT)).toString(); } - + @Override // Can't do much about this instance if used via the RPC layer. Any downstream // users can however avoid using this method. @@ -125,7 +125,7 @@ public void readFields(DataInput in) throws IOException { taskId = TezTaskID.readTezTaskID(in); super.readFields(in); } - + public static TezTaskAttemptID readTezTaskAttemptID(DataInput in) throws IOException { TezTaskID taskID = TezTaskID.readTezTaskID(in); int attemptIdInt = TezID.readID(in); @@ -163,5 +163,4 @@ public static TezTaskAttemptID fromString(String taIdStr) { } return null; } - } diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java index 08310f3dfc..f505406b20 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java +++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezTaskID.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -44,7 +44,7 @@ public class TezTaskID extends TezID implements VertexIDAware { public static final String TASK = "task"; private final int serializingHash; - + static final ThreadLocal tezTaskIdFormat = new ThreadLocal() { @Override public FastNumberFormat initialValue() { @@ -73,7 +73,7 @@ private TezTaskID(TezVertexID vertexID, int id) { this.vertexId = vertexID; this.serializingHash = getHashCode(true); } - + public int getSerializingHash() { return serializingHash; } @@ -89,20 +89,20 @@ public boolean equals(Object o) { if (!super.equals(o)) return false; - TezTaskID that = (TezTaskID)o; + TezTaskID that = (TezTaskID) o; return this.vertexId.equals(that.vertexId); } /**Compare TaskInProgressIds by first jobIds, then by tip numbers and type.*/ @Override public int compareTo(TezID o) { - TezTaskID that = (TezTaskID)o; + TezTaskID that = (TezTaskID) o; int vertexComp = this.vertexId.compareTo(that.vertexId); - if(vertexComp == 0) { + if (vertexComp == 0) { return this.id - that.id; - } - else return vertexComp; + } else return vertexComp; } + @Override public String toString() { return appendTo(new StringBuilder(TASK)).toString(); @@ -139,7 +139,7 @@ public void readFields(DataInput in) throws IOException { vertexId = TezVertexID.readTezVertexID(in); super.readFields(in); } - + public static TezTaskID readTezTaskID(DataInput in) throws IOException { TezVertexID vertexID = TezVertexID.readTezVertexID(in); int taskIdInt = TezID.readID(in); @@ -166,13 +166,12 @@ public static TezTaskID fromString(String taskIdStr) { int id = Integer.parseInt(taskIdStr.substring(pos5 + 1)); return TezTaskID.getInstance( - TezVertexID.getInstance( - TezDAGID.getInstance(rmId, appId, dagId), - vId), id); + TezVertexID.getInstance( + TezDAGID.getInstance(rmId, appId, dagId), + vId), id); } catch (Exception e) { e.printStackTrace(); } return null; } - } diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/TezVertexID.java b/tez-common/src/main/java/org/apache/tez/dag/records/TezVertexID.java index f7becc250f..e7b3c53b21 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/records/TezVertexID.java +++ b/tez-common/src/main/java/org/apache/tez/dag/records/TezVertexID.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -90,14 +90,14 @@ public boolean equals(Object o) { if (!super.equals(o)) return false; - TezVertexID that = (TezVertexID)o; + TezVertexID that = (TezVertexID) o; return this.dagId.equals(that.dagId); } /**Compare TaskInProgressIds by first jobIds, then by tip numbers and type.*/ @Override public int compareTo(TezID o) { - TezVertexID that = (TezVertexID)o; + TezVertexID that = (TezVertexID) o; return this.dagId.compareTo(that.dagId); } @@ -113,7 +113,7 @@ public void readFields(DataInput in) throws IOException { dagId = TezDAGID.readTezDAGID(in); super.readFields(in); } - + public static TezVertexID readTezVertexID(DataInput in) throws IOException { TezDAGID dagID = TezDAGID.readTezDAGID(in); int vertexIdInt = TezID.readID(in); @@ -153,8 +153,8 @@ public static TezVertexID fromString(String vertexIdStr) { int dagId = Integer.parseInt(vertexIdStr.substring(pos3 + 1, pos4)); int id = Integer.parseInt(vertexIdStr.substring(pos4 + 1)); return TezVertexID.getInstance( - TezDAGID.getInstance(rmId, appId, dagId), - id); + TezDAGID.getInstance(rmId, appId, dagId), + id); } catch (Exception e) { e.printStackTrace(); } diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/VertexIDAware.java b/tez-common/src/main/java/org/apache/tez/dag/records/VertexIDAware.java index 01bbe859b6..a0d3b3de67 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/records/VertexIDAware.java +++ b/tez-common/src/main/java/org/apache/tez/dag/records/VertexIDAware.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,10 +19,10 @@ package org.apache.tez.dag.records; public interface VertexIDAware extends DAGIDAware { - TezVertexID getVertexID(); + TezVertexID getVertexID(); - @Override - default TezDAGID getDAGID() { - return getVertexID().getDAGID(); - } + @Override + default TezDAGID getDAGID() { + return getVertexID().getDAGID(); + } } diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/VertexIdentifierImpl.java b/tez-common/src/main/java/org/apache/tez/dag/records/VertexIdentifierImpl.java index 83b503203c..af8452368b 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/records/VertexIdentifierImpl.java +++ b/tez-common/src/main/java/org/apache/tez/dag/records/VertexIdentifierImpl.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,7 +26,7 @@ public class VertexIdentifierImpl implements VertexIdentifier { private final DagIdentifier dagIdentifier; private final TezVertexID vertexId; private final String vertexName; - + public VertexIdentifierImpl(String dagName, String vertexName, TezVertexID vertexId) { this.vertexId = vertexId; this.vertexName = vertexName; @@ -42,34 +42,33 @@ public String getName() { public int getIdentifier() { return vertexId.getId(); } - + @Override public DagIdentifier getDagIdentifier() { return dagIdentifier; } - + @Override public boolean equals(Object o) { if (this == o) { return true; } - if(o == null) { + if (o == null) { return false; } if (o.getClass() == this.getClass()) { VertexIdentifierImpl other = (VertexIdentifierImpl) o; return this.vertexId.equals(other.vertexId); - } - else { + } else { return false; } } - + @Override public String toString() { return dagIdentifier.toString() + " Vertex: " + vertexName + ":[" + getIdentifier() + "]"; } - + @Override public int hashCode() { return vertexId.hashCode(); diff --git a/tez-common/src/main/java/org/apache/tez/dag/records/package-info.java b/tez-common/src/main/java/org/apache/tez/dag/records/package-info.java index 319ab6b99a..aa1ee3b54a 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/records/package-info.java +++ b/tez-common/src/main/java/org/apache/tez/dag/records/package-info.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,4 +19,4 @@ @Private package org.apache.tez.dag.records; -import org.apache.hadoop.classification.InterfaceAudience.Private; \ No newline at end of file +import org.apache.hadoop.classification.InterfaceAudience.Private; diff --git a/tez-common/src/main/java/org/apache/tez/dag/utils/RelocalizationUtils.java b/tez-common/src/main/java/org/apache/tez/dag/utils/RelocalizationUtils.java index 84a9474a39..5e44cfcb0a 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/utils/RelocalizationUtils.java +++ b/tez-common/src/main/java/org/apache/tez/dag/utils/RelocalizationUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -37,9 +37,9 @@ @InterfaceAudience.Private public class RelocalizationUtils { - + public static List processAdditionalResources(Map additionalResources, - Configuration conf, String destDir) throws IOException, TezException { + Configuration conf, String destDir) throws IOException, TezException { if (additionalResources == null || additionalResources.isEmpty()) { return Collections.emptyList(); } diff --git a/tez-common/src/main/java/org/apache/tez/dag/utils/package-info.java b/tez-common/src/main/java/org/apache/tez/dag/utils/package-info.java index 0cfda8411a..d14b769c13 100644 --- a/tez-common/src/main/java/org/apache/tez/dag/utils/package-info.java +++ b/tez-common/src/main/java/org/apache/tez/dag/utils/package-info.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,4 +19,4 @@ @Private package org.apache.tez.dag.utils; -import org.apache.hadoop.classification.InterfaceAudience.Private; \ No newline at end of file +import org.apache.hadoop.classification.InterfaceAudience.Private; diff --git a/tez-common/src/main/java/org/apache/tez/runtime/common/resources/InitialMemoryAllocator.java b/tez-common/src/main/java/org/apache/tez/runtime/common/resources/InitialMemoryAllocator.java index 6910f239c0..b9c0bda65e 100644 --- a/tez-common/src/main/java/org/apache/tez/runtime/common/resources/InitialMemoryAllocator.java +++ b/tez-common/src/main/java/org/apache/tez/runtime/common/resources/InitialMemoryAllocator.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,8 +21,6 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configurable; - - /** * Used to balance memory requests before a task starts executing. */ @@ -42,6 +40,5 @@ public interface InitialMemoryAllocator extends Configurable { * same order of the requests. */ public abstract Iterable assignMemory(long availableForAllocation, int numTotalInputs, - int numTotalOutputs, Iterable requests); - -} \ No newline at end of file + int numTotalOutputs, Iterable requests); +} diff --git a/tez-common/src/main/java/org/apache/tez/runtime/common/resources/InitialMemoryRequestContext.java b/tez-common/src/main/java/org/apache/tez/runtime/common/resources/InitialMemoryRequestContext.java index da361a1e59..f0fc6fa4ae 100644 --- a/tez-common/src/main/java/org/apache/tez/runtime/common/resources/InitialMemoryRequestContext.java +++ b/tez-common/src/main/java/org/apache/tez/runtime/common/resources/InitialMemoryRequestContext.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,7 +22,6 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; - @Private public class InitialMemoryRequestContext { @@ -38,7 +37,7 @@ public static enum ComponentType { private String componentVertexName; public InitialMemoryRequestContext(long requestedSize, String componentClassName, - ComponentType componentType, String componentVertexName) { + ComponentType componentType, String componentVertexName) { Objects.requireNonNull(componentClassName, "componentClassName is null"); Objects.requireNonNull(componentType, "componentType is null"); Objects.requireNonNull(componentVertexName, "componentVertexName is null"); @@ -63,5 +62,4 @@ public ComponentType getComponentType() { public String getComponentVertexName() { return componentVertexName; } - } diff --git a/tez-common/src/main/java/org/apache/tez/runtime/common/resources/package-info.java b/tez-common/src/main/java/org/apache/tez/runtime/common/resources/package-info.java index 49689e09ec..65faa8f7e5 100644 --- a/tez-common/src/main/java/org/apache/tez/runtime/common/resources/package-info.java +++ b/tez-common/src/main/java/org/apache/tez/runtime/common/resources/package-info.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,4 +19,4 @@ @Private package org.apache.tez.runtime.common.resources; -import org.apache.hadoop.classification.InterfaceAudience.Private; \ No newline at end of file +import org.apache.hadoop.classification.InterfaceAudience.Private; diff --git a/tez-common/src/main/java/org/apache/tez/util/FastNumberFormat.java b/tez-common/src/main/java/org/apache/tez/util/FastNumberFormat.java index f22fc64b46..94b79984e6 100644 --- a/tez-common/src/main/java/org/apache/tez/util/FastNumberFormat.java +++ b/tez-common/src/main/java/org/apache/tez/util/FastNumberFormat.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,10 +36,10 @@ public StringBuilder format(long source, StringBuilder sb) { int left = MAX_COUNT; if (source < 0) { sb.append('-'); - source = - source; + source = -source; } while (source > 0) { - digits[--left] = (char)('0' + (source % 10)); + digits[--left] = (char) ('0' + (source % 10)); source /= 10; } while (MAX_COUNT - left < minimumIntegerDigits) { diff --git a/tez-common/src/main/java/org/apache/tez/util/StopWatch.java b/tez-common/src/main/java/org/apache/tez/util/StopWatch.java index c9a573e88f..58c203dd71 100644 --- a/tez-common/src/main/java/org/apache/tez/util/StopWatch.java +++ b/tez-common/src/main/java/org/apache/tez/util/StopWatch.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -82,7 +82,6 @@ public StopWatch reset() { */ public long now(TimeUnit timeUnit) { return timeUnit.convert(now(), TimeUnit.NANOSECONDS); - } /** diff --git a/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java b/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java index 4a11a8aaa9..0231d9ad9f 100644 --- a/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java +++ b/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -47,18 +47,20 @@ public TezMxBeanResourceCalculator(String root) { super(root); runtime = Runtime.getRuntime(); osBean = ManagementFactory.getOperatingSystemMXBean(); - } - @Override public void updateProcessTree() { + @Override + public void updateProcessTree() { //nothing needs to be done as the data is read from OS mbeans. } - @Override public String getProcessTreeDump() { + @Override + public String getProcessTreeDump() { return ""; } - @Override public long getVirtualMemorySize(int olderThanAge) { + @Override + public long getVirtualMemorySize(int olderThanAge) { try { return (Long) getCommittedVirtualMemorySize.invoke(osBean); } catch (IllegalArgumentException e) { @@ -70,16 +72,18 @@ public TezMxBeanResourceCalculator(String root) { } } - @Override public long getRssMemorySize(int olderThanAge) { + @Override + public long getRssMemorySize(int olderThanAge) { //Not supported directly (RSS ~= memory consumed by JVM from Xmx) return runtime.totalMemory(); } - @Override public long getCumulativeCpuTime() { + @Override + public long getCumulativeCpuTime() { //convert to milliseconds try { return TimeUnit.MILLISECONDS.convert( - (Long) getProcessCpuTime.invoke(osBean), TimeUnit.NANOSECONDS); + (Long) getProcessCpuTime.invoke(osBean), TimeUnit.NANOSECONDS); } catch (InvocationTargetException e) { return -1; } catch (IllegalArgumentException e) { @@ -89,7 +93,8 @@ public TezMxBeanResourceCalculator(String root) { } } - @Override public boolean checkPidPgrpidForMatch() { + @Override + public boolean checkPidPgrpidForMatch() { return true; } @@ -98,9 +103,9 @@ public float getCpuUsagePercent() { //Returning -1 to indicate, this feature is not yet supported. return -1; } - + private static Method getMxBeanMethod(String methodName) { - // New Method to support IBM and Oracle/OpenJDK JDK with OperatingSystemMXBean + // New Method to support IBM and Oracle/OpenJDK JDK with OperatingSystemMXBean final String JAVA_VENDOR_NAME = System.getProperty("java.vendor"); final boolean IBM_JAVA = JAVA_VENDOR_NAME.contains("IBM"); try { @@ -110,7 +115,7 @@ private static Method getMxBeanMethod(String methodName) { } else { mbeanClazz = Class.forName("com.sun.management.OperatingSystemMXBean"); } - if (IBM_JAVA){ + if (IBM_JAVA) { if (methodName.equals("getCommittedVirtualMemorySize")) { methodName = "getProcessVirtualMemorySize"; } @@ -119,7 +124,7 @@ private static Method getMxBeanMethod(String methodName) { } } final Method method = mbeanClazz - .getMethod(methodName); + .getMethod(methodName); return method; } catch (ClassNotFoundException e) { return null; diff --git a/tez-common/src/test/java/org/apache/tez/common/DrainDispatcher.java b/tez-common/src/test/java/org/apache/tez/common/DrainDispatcher.java index fd1fc0aa90..658d8a42a2 100644 --- a/tez-common/src/test/java/org/apache/tez/common/DrainDispatcher.java +++ b/tez-common/src/test/java/org/apache/tez/common/DrainDispatcher.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -53,7 +53,7 @@ public void register(Class eventType, LOG.info("Registering " + eventType + " for " + handler.getClass()); if (registeredHandler == null) { eventHandlers.put(eventType, handler); - } else if (!(registeredHandler instanceof MultiListenerHandler)){ + } else if (!(registeredHandler instanceof MultiListenerHandler)) { /* for multiple listeners of an event add the multiple listener handler */ MultiListenerHandler multiHandler = new MultiListenerHandler(); multiHandler.addHandler(registeredHandler); @@ -62,7 +62,7 @@ public void register(Class eventType, } else { /* already a multilistener, just add to it */ MultiListenerHandler multiHandler - = (MultiListenerHandler) registeredHandler; + = (MultiListenerHandler) registeredHandler; multiHandler.addHandler(handler); } } diff --git a/tez-common/src/test/java/org/apache/tez/common/MockDNSToSwitchMapping.java b/tez-common/src/test/java/org/apache/tez/common/MockDNSToSwitchMapping.java index afcd687356..98ba580337 100644 --- a/tez-common/src/test/java/org/apache/tez/common/MockDNSToSwitchMapping.java +++ b/tez-common/src/test/java/org/apache/tez/common/MockDNSToSwitchMapping.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-common/src/test/java/org/apache/tez/common/TestAsyncDispatcher.java b/tez-common/src/test/java/org/apache/tez/common/TestAsyncDispatcher.java index bcd1c5f2f7..dea6851d3e 100644 --- a/tez-common/src/test/java/org/apache/tez/common/TestAsyncDispatcher.java +++ b/tez-common/src/test/java/org/apache/tez/common/TestAsyncDispatcher.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -29,6 +29,7 @@ public class TestAsyncDispatcher { static class CountDownEventHandler { static CountDownLatch latch; + public void handle() { latch.countDown(); try { @@ -39,54 +40,62 @@ public void handle() { } } - public enum TestEventType1 { TYPE1 } + public enum TestEventType1 {TYPE1} + public class TestEvent1 extends AbstractEvent { public TestEvent1(TestEventType1 type) { super(type); } } + class TestEventHandler1 extends CountDownEventHandler implements EventHandler { @Override public void handle(TestEvent1 event) { handle(); } } - public enum TestEventType2 { TYPE2 } + + public enum TestEventType2 {TYPE2} + public class TestEvent2 extends AbstractEvent { public TestEvent2(TestEventType2 type) { super(type); } } + class TestEventHandler2 extends CountDownEventHandler implements EventHandler { @Override public void handle(TestEvent2 event) { handle(); } } - public enum TestEventType3 { TYPE3 } + + public enum TestEventType3 {TYPE3} + public class TestEvent3 extends AbstractEvent { public TestEvent3(TestEventType3 type) { super(type); } } + class TestEventHandler3 extends CountDownEventHandler implements EventHandler { @Override public void handle(TestEvent3 event) { handle(); } } - + @SuppressWarnings("unchecked") - @Test (timeout=5000) + @Test(timeout = 5000) public void testBasic() throws Exception { CountDownLatch latch = new CountDownLatch(4); CountDownEventHandler.latch = latch; - + AsyncDispatcher central = new AsyncDispatcher("Type1"); central.register(TestEventType1.class, new TestEventHandler1()); central.registerAndCreateDispatcher(TestEventType2.class, new TestEventHandler2(), "Type2"); central.registerAndCreateDispatcher(TestEventType3.class, new TestEventHandler3(), "Type3"); - + central.init(new Configuration()); central.start(); central.getEventHandler().handle(new TestEvent1(TestEventType1.TYPE1)); @@ -96,8 +105,8 @@ public void testBasic() throws Exception { latch.await(); central.close(); } - - @Test (timeout=5000) + + @Test(timeout = 5000) public void testMultipleRegisterFail() throws Exception { AsyncDispatcher central = new AsyncDispatcher("Type1"); try { @@ -109,7 +118,7 @@ public void testMultipleRegisterFail() throws Exception { } finally { central.close(); } - + central = new AsyncDispatcher("Type1"); try { central.registerAndCreateDispatcher(TestEventType1.class, new TestEventHandler2(), "Type2"); diff --git a/tez-common/src/test/java/org/apache/tez/common/TestAsyncDispatcherConcurrent.java b/tez-common/src/test/java/org/apache/tez/common/TestAsyncDispatcherConcurrent.java index 1fa8123956..4059a202ba 100644 --- a/tez-common/src/test/java/org/apache/tez/common/TestAsyncDispatcherConcurrent.java +++ b/tez-common/src/test/java/org/apache/tez/common/TestAsyncDispatcherConcurrent.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -29,6 +29,7 @@ public class TestAsyncDispatcherConcurrent { static class CountDownEventHandler { static CountDownLatch latch; + static void init(CountDownLatch latch) { CountDownEventHandler.latch = latch; } @@ -37,7 +38,7 @@ static void checkParallelCountersDoneAndFinish() throws Exception { latch.countDown(); latch.await(); } - + public void handle() { latch.countDown(); try { @@ -48,43 +49,52 @@ public void handle() { } } - public enum TestEventType1 { TYPE1 } + public enum TestEventType1 {TYPE1} + public class TestEvent1 extends TezAbstractEvent { final int hash; + public TestEvent1(TestEventType1 type, int hash) { super(type); this.hash = hash; } - + @Override public int getSerializingHash() { return hash; } } + class TestEventHandler1 extends CountDownEventHandler implements EventHandler { @Override public void handle(TestEvent1 event) { handle(); } } - public enum TestEventType2 { TYPE2 } + + public enum TestEventType2 {TYPE2} + public class TestEvent2 extends TezAbstractEvent { public TestEvent2(TestEventType2 type) { super(type); } } + class TestEventHandler2 extends CountDownEventHandler implements EventHandler { @Override public void handle(TestEvent2 event) { handle(); } } - public enum TestEventType3 { TYPE3 } + + public enum TestEventType3 {TYPE3} + public class TestEvent3 extends TezAbstractEvent { public TestEvent3(TestEventType3 type) { super(type); } } + class TestEventHandler3 extends CountDownEventHandler implements EventHandler { @Override public void handle(TestEvent3 event) { @@ -92,16 +102,16 @@ public void handle(TestEvent3 event) { } } - @Test (timeout=5000) + @Test(timeout = 5000) public void testBasic() throws Exception { CountDownLatch latch = new CountDownLatch(4); CountDownEventHandler.init(latch); - + AsyncDispatcher central = new AsyncDispatcher("Type1"); central.register(TestEventType1.class, new TestEventHandler1()); central.registerAndCreateDispatcher(TestEventType2.class, new TestEventHandler2(), "Type2", 1); central.registerAndCreateDispatcher(TestEventType3.class, new TestEventHandler3(), "Type3", 1); - + central.init(new Configuration()); central.start(); // 3 threads in different dispatchers will handle 3 events @@ -112,15 +122,15 @@ public void testBasic() throws Exception { CountDownEventHandler.checkParallelCountersDoneAndFinish(); central.close(); } - - @Test (timeout=5000) + + @Test(timeout = 5000) public void testMultiThreads() throws Exception { CountDownLatch latch = new CountDownLatch(4); CountDownEventHandler.init(latch); - + AsyncDispatcherConcurrent central = new AsyncDispatcherConcurrent("Type1", 1); central.registerAndCreateDispatcher(TestEventType1.class, new TestEventHandler1(), "Type1", 3); - + central.init(new Configuration()); central.start(); // 3 threads in the same dispatcher will handle 3 events @@ -131,8 +141,8 @@ public void testMultiThreads() throws Exception { CountDownEventHandler.checkParallelCountersDoneAndFinish(); central.close(); } - - @Test (timeout=5000) + + @Test(timeout = 5000) public void testMultipleRegisterFail() throws Exception { AsyncDispatcher central = new AsyncDispatcher("Type1"); try { @@ -144,7 +154,7 @@ public void testMultipleRegisterFail() throws Exception { } finally { central.close(); } - + central = new AsyncDispatcher("Type1"); try { central.registerAndCreateDispatcher(TestEventType1.class, new TestEventHandler2(), "Type2", 1); @@ -155,7 +165,7 @@ public void testMultipleRegisterFail() throws Exception { } finally { central.close(); } - + central = new AsyncDispatcher("Type1"); try { central.registerAndCreateDispatcher(TestEventType1.class, new TestEventHandler2(), "Type2", 1); @@ -166,7 +176,7 @@ public void testMultipleRegisterFail() throws Exception { } finally { central.close(); } - + central = new AsyncDispatcher("Type1"); try { central.registerAndCreateDispatcher(TestEventType1.class, new TestEventHandler2(), "Type2"); @@ -177,7 +187,7 @@ public void testMultipleRegisterFail() throws Exception { } finally { central.close(); } - + central = new AsyncDispatcher("Type1"); try { AsyncDispatcherConcurrent concDispatcher = central.registerAndCreateDispatcher( diff --git a/tez-common/src/test/java/org/apache/tez/common/TestTezSharedExecutor.java b/tez-common/src/test/java/org/apache/tez/common/TestTezSharedExecutor.java index 9ea07efcba..6b7b77f8e6 100644 --- a/tez-common/src/test/java/org/apache/tez/common/TestTezSharedExecutor.java +++ b/tez-common/src/test/java/org/apache/tez/common/TestTezSharedExecutor.java @@ -42,9 +42,11 @@ public class TestTezSharedExecutor { private static class Sleep implements Runnable { private final long sleepTime; + Sleep(long sleepTime) { this.sleepTime = sleepTime; } + @Override public void run() { try { @@ -57,9 +59,11 @@ public void run() { private static class Wait implements Runnable { private final CountDownLatch latch; + Wait(CountDownLatch latch) { this.latch = latch; } + @Override public void run() { try { @@ -72,12 +76,14 @@ public void run() { private static class Counter implements Runnable { private final AtomicInteger counter; + Counter(ConcurrentHashMap map, String tag) { if (!map.contains(tag)) { map.putIfAbsent(tag, new AtomicInteger(0)); } this.counter = map.get(tag); } + @Override public void run() { counter.getAndIncrement(); @@ -87,10 +93,12 @@ public void run() { private static class Appender implements Runnable { private final Collection collection; private final T obj; + Appender(Collection collection, T obj) { this.collection = collection; this.obj = obj; } + @Override public void run() { collection.add(obj); @@ -99,9 +107,11 @@ public void run() { private static class Runner implements Runnable { private Runnable[] runnables; - Runner(Runnable ... runnables) { + + Runner(Runnable... runnables) { this.runnables = runnables; } + @Override public void run() { for (Runnable runnable : runnables) { @@ -123,7 +133,7 @@ public void cleanup() { sharedExecutor = null; } - @Test(timeout=10000) + @Test(timeout = 10000) public void testSimpleExecution() throws Exception { ConcurrentHashMap map = new ConcurrentHashMap<>(); @@ -158,7 +168,7 @@ public Object call() throws Exception { } } - @Test(timeout=10000) + @Test(timeout = 10000) public void testAwaitTermination() throws Exception { ExecutorService service = sharedExecutor.createExecutorService(1, "await-termination"); CountDownLatch latch = new CountDownLatch(1); @@ -179,7 +189,7 @@ public void testAwaitTermination() throws Exception { Assert.assertTrue(service.isShutdown()); } - @Test(timeout=10000) + @Test(timeout = 10000) public void testSerialExecution() throws Exception { ExecutorService service = sharedExecutor.createExecutorService(1, "serial-test"); CountDownLatch latch = new CountDownLatch(1); @@ -208,7 +218,7 @@ public void testSerialExecution() throws Exception { Assert.assertEquals(Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9), list); } - @Test(timeout=10000) + @Test(timeout = 10000) public void testParallelExecution() throws Exception { ConcurrentHashMap map = new ConcurrentHashMap<>(); @@ -236,5 +246,4 @@ public void testParallelExecution() throws Exception { services[1].submit(new Counter(map, "test1")).get(); Assert.assertEquals(expectedCounts[1] + 1, map.get("test1").get()); } - } diff --git a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java index d599cafd76..fa76c22073 100644 --- a/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java +++ b/tez-common/src/test/java/org/apache/tez/common/TestTezUtils.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -42,7 +42,7 @@ public class TestTezUtils { - @Test (timeout=2000) + @Test(timeout = 2000) public void testByteStringToAndFromConf() throws IOException { Configuration conf = getConf(); Assert.assertEquals(conf.size(), 6); @@ -77,7 +77,7 @@ private ByteString createByteString(Configuration conf, String largeValue) throw return TezUtils.createByteStringFromConf(conf); } - @Test (timeout=20000) + @Test(timeout = 20000) public void testByteStringToAndFromLargeConf() throws IOException { Configuration conf = getConf(); String largeValue = constructLargeValue(); @@ -90,7 +90,7 @@ public void testByteStringToAndFromLargeConf() throws IOException { Assert.assertEquals(conf.get("testLargeValue"), largeValue); } - @Test (timeout=20000) + @Test(timeout = 20000) public void testByteStringAddToLargeConf() throws IOException { Configuration conf = getConf(); String largeValue = constructLargeValue(); @@ -103,7 +103,7 @@ public void testByteStringAddToLargeConf() throws IOException { Assert.assertEquals(conf.get("testLargeValue"), largeValue); } - @Test (timeout=2000) + @Test(timeout = 2000) public void testPayloadToAndFromConf() throws IOException { Configuration conf = getConf(); Assert.assertEquals(conf.size(), 6); @@ -115,7 +115,7 @@ public void testPayloadToAndFromConf() throws IOException { checkConf(conf); } - @Test (timeout=2000) + @Test(timeout = 2000) public void testCleanVertexName() { String testString = "special characters & spaces and longer than " + TezUtilsInternal.MAX_VERTEX_NAME_LENGTH + " characters"; @@ -126,7 +126,7 @@ public void testCleanVertexName() { Assert.assertTrue(cleaned.matches("\\w+")); } - @Test (timeout=2000) + @Test(timeout = 2000) public void testBitSetToByteArray() { BitSet bitSet = createBitSet(0); byte[] bytes = TezUtilsInternal.toByteArray(bitSet); @@ -137,7 +137,7 @@ public void testBitSetToByteArray() { Assert.assertEquals(bytes.length, (bitSet.length() + 7) / 8); } - @Test (timeout=2000) + @Test(timeout = 2000) public void testBitSetFromByteArray() { BitSet bitSet = createBitSet(0); byte[] bytes = TezUtilsInternal.toByteArray(bitSet); @@ -148,19 +148,19 @@ public void testBitSetFromByteArray() { bytes = TezUtilsInternal.toByteArray(bitSet); Assert.assertEquals(TezUtilsInternal.fromByteArray(bytes).cardinality(), bitSet.cardinality()); Assert.assertTrue(TezUtilsInternal.fromByteArray(bytes).equals(bitSet)); - + bitSet = createBitSet(1000); bytes = TezUtilsInternal.toByteArray(bitSet); Assert.assertEquals(TezUtilsInternal.fromByteArray(bytes).cardinality(), bitSet.cardinality()); Assert.assertTrue(TezUtilsInternal.fromByteArray(bytes).equals(bitSet)); } - @Test (timeout=2000) + @Test(timeout = 2000) public void testBitSetConversion() { - for (int i = 0 ; i < 16 ; i++) { + for (int i = 0; i < 16; i++) { BitSet bitSet = createBitSetWithSingleEntry(i); byte[] bytes = TezUtilsInternal.toByteArray(bitSet); - + BitSet deseraialized = TezUtilsInternal.fromByteArray(bytes); Assert.assertEquals(bitSet, deseraialized); Assert.assertEquals(bitSet.cardinality(), deseraialized.cardinality()); @@ -172,7 +172,7 @@ private BitSet createBitSet(int size) { BitSet bitSet = new BitSet(); int bitsToEnable = (int) (size * 0.1); Random rnd = new Random(); - for(int i = 0;i < bitsToEnable;i++) { + for (int i = 0; i < bitsToEnable; i++) { bitSet.set(rnd.nextInt(size)); } return bitSet; @@ -206,7 +206,6 @@ private void checkConf(Configuration conf) { Assert.assertEquals(tmp[0], "S1"); Assert.assertEquals(tmp[1], "S2"); Assert.assertEquals(tmp[2], "S3"); - } private void checkJSONConfigObj(JSONObject confObject) throws JSONException { @@ -219,7 +218,7 @@ private void checkJSONConfigObj(JSONObject confObject) throws JSONException { Assert.assertEquals("S1,S2,S3", confObject.getString("test6")); } - @Test (timeout=2000) + @Test(timeout = 2000) public void testConvertToHistoryText() throws JSONException { Configuration conf = getConf(); @@ -244,10 +243,9 @@ public void testConvertToHistoryText() throws JSONException { Assert.assertTrue(jsonObject.has(ATSConstants.CONFIG)); confObject = jsonObject.getJSONObject("config"); checkJSONConfigObj(confObject); - } - @Test (timeout=2000) + @Test(timeout = 2000) public void testConvertToHistoryTextWithReplaceVars() throws JSONException { Configuration conf = getConf(); conf.set("user", "user1"); @@ -281,17 +279,15 @@ public void testConvertToHistoryTextWithReplaceVars() throws JSONException { checkJSONConfigObj(confObject); Assert.assertEquals("user1", confObject.getString("user")); Assert.assertEquals(location, confObject.getString("location")); - } @Test(timeout = 5000) public void testPopulateConfProtoFromEntries() { - Map map = new HashMap<>(); - map.put("nonNullKey", "value"); - map.put("nullKey", null); - DAGProtos.ConfigurationProto.Builder confBuilder = DAGProtos.ConfigurationProto.newBuilder(); - TezUtils.populateConfProtoFromEntries(map.entrySet(), confBuilder); - assertEquals(confBuilder.getConfKeyValuesList().size(), 1); + Map map = new HashMap<>(); + map.put("nonNullKey", "value"); + map.put("nullKey", null); + DAGProtos.ConfigurationProto.Builder confBuilder = DAGProtos.ConfigurationProto.newBuilder(); + TezUtils.populateConfProtoFromEntries(map.entrySet(), confBuilder); + assertEquals(confBuilder.getConfKeyValuesList().size(), 1); } - } diff --git a/tez-common/src/test/java/org/apache/tez/dag/records/TestTezIds.java b/tez-common/src/test/java/org/apache/tez/dag/records/TestTezIds.java index 10b62b2bd1..58aef12521 100644 --- a/tez-common/src/test/java/org/apache/tez/dag/records/TestTezIds.java +++ b/tez-common/src/test/java/org/apache/tez/dag/records/TestTezIds.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -17,7 +17,6 @@ package org.apache.tez.dag.records; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -108,7 +107,7 @@ public void testIdStringify() { verifyAttemptId(taIdStr, taId); } - @Test(timeout=5000) + @Test(timeout = 5000) public void testInvalidDagIds() { String dagIdStr = "aaa_111_1_1"; TezDAGID dagId; @@ -142,10 +141,8 @@ public void testInvalidDagIds() { } catch (IllegalArgumentException e) { Assert.assertTrue(e.getMessage().contains("Error while parsing")); } - } - public void testGetGroupIds() { ApplicationId appId = ApplicationId.newInstance(0, 1); int numDagsPerGroup = 5; @@ -170,5 +167,4 @@ public void testGetGroupIds() { } } } - } diff --git a/tez-common/src/test/java/org/apache/tez/util/TestNumberFormat.java b/tez-common/src/test/java/org/apache/tez/util/TestNumberFormat.java index c2f1185294..288b36a138 100644 --- a/tez-common/src/test/java/org/apache/tez/util/TestNumberFormat.java +++ b/tez-common/src/test/java/org/apache/tez/util/TestNumberFormat.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -32,7 +32,7 @@ public void testLongWithPadding() throws Exception { numberFormat.setGroupingUsed(false); numberFormat.setMinimumIntegerDigits(6); long[] testLongs = {1, 23, 456, 7890, 12345, 678901, 2345689, 0, -0, -1, -23, -456, -7890, -12345, -678901, -2345689}; - for (long l: testLongs) { + for (long l : testLongs) { Assert.assertEquals("Number formats should be equal", numberFormat.format(l), fastNumberFormat.format(l)); } } diff --git a/tez-common/src/test/java/org/apache/tez/util/TestStopWatch.java b/tez-common/src/test/java/org/apache/tez/util/TestStopWatch.java index b8523fd09f..0938e07b1d 100644 --- a/tez-common/src/test/java/org/apache/tez/util/TestStopWatch.java +++ b/tez-common/src/test/java/org/apache/tez/util/TestStopWatch.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -54,5 +54,4 @@ public void testExceptions() throws Exception { e instanceof IllegalStateException); } } - } diff --git a/tez-common/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java b/tez-common/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java index 803118a4f9..8c4c6107b7 100644 --- a/tez-common/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java +++ b/tez-common/src/test/java/org/apache/tez/util/TestTezMxBeanResourceCalculator.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -48,7 +48,7 @@ public void teardown() { resourceCalculator = null; } - @Test(timeout=5000) + @Test(timeout = 5000) public void testResourceCalculator() { Assert.assertTrue(resourceCalculator instanceof TezMxBeanResourceCalculator); Assert.assertTrue(resourceCalculator.getCumulativeCpuTime() > 0); @@ -57,5 +57,4 @@ public void testResourceCalculator() { Assert.assertTrue(resourceCalculator.getProcessTreeDump().equals("")); Assert.assertTrue(resourceCalculator.checkPidPgrpidForMatch()); } - } diff --git a/tez-dag/src/main/java/org/apache/tez/Utils.java b/tez-dag/src/main/java/org/apache/tez/Utils.java index b352334f44..e4a2d62172 100644 --- a/tez-dag/src/main/java/org/apache/tez/Utils.java +++ b/tez-dag/src/main/java/org/apache/tez/Utils.java @@ -15,6 +15,7 @@ package org.apache.tez; import javax.annotation.Nullable; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.yarn.event.Event; import org.apache.tez.dag.api.records.DAGProtos; @@ -109,21 +110,22 @@ public static void processNonFatalServiceErrorReport(String entityString, /** * Generate a visualization file. - * @param dag DAG. - * @param dagPB DAG plan. - * @param logDirs directories where the file will be written. + * + * @param dag DAG. + * @param dagPB DAG plan. + * @param logDirs directories where the file will be written. * @param scheduler scheduler that will provide the priorities * of the vertexes. */ public static void generateDAGVizFile(final DAG dag, - final DAGProtos.DAGPlan dagPB, - final String[] logDirs, final @Nullable DAGScheduler scheduler) { + final DAGProtos.DAGPlan dagPB, + final String[] logDirs, final @Nullable DAGScheduler scheduler) { TezDAGID dagId = dag.getID(); HashMap nameToVertex = null; if (scheduler != null) { nameToVertex = new HashMap<>(dag.getVertices().size()); - for (Vertex v: dag.getVertices().values()) { + for (Vertex v : dag.getVertices().values()) { nameToVertex.put(v.getName(), v); } } @@ -132,7 +134,7 @@ public static void generateDAGVizFile(final DAG dag, for (DAGProtos.VertexPlan vertexPlan : dagPB.getVertexList()) { StringBuilder nodeLabel = new StringBuilder( sanitizeLabelForViz(vertexPlan.getName()) - + "[" + getShortClassName( + + "[" + getShortClassName( vertexPlan.getProcessorDescriptor().getClassName())); if (scheduler != null) { @@ -140,7 +142,7 @@ public static void generateDAGVizFile(final DAG dag, if (vertex != null) { try { int priority = (scheduler.getPriorityLowLimit(dag, vertex) - + scheduler.getPriorityHighLimit(dag,vertex)) / 2; + + scheduler.getPriorityHighLimit(dag, vertex)) / 2; nodeLabel.append(", priority=").append(priority).append("]"); } catch (UnsupportedOperationException e) { LOG.info("The DAG graphviz file with priorities will not" @@ -157,29 +159,29 @@ public static void generateDAGVizFile(final DAG dag, : vertexPlan.getInputsList()) { Graph.Node inputNode = graph.getNode( sanitizeLabelForViz(vertexPlan.getName()) - + "_" + sanitizeLabelForViz(input.getName())); + + "_" + sanitizeLabelForViz(input.getName())); inputNode.setLabel(sanitizeLabelForViz(vertexPlan.getName()) + "[" + sanitizeLabelForViz(input.getName()) + "]"); inputNode.setShape("box"); inputNode.addEdge(n, "Input" + " [inputClass=" + getShortClassName( - input.getIODescriptor().getClassName()) + input.getIODescriptor().getClassName()) + ", initializer=" + getShortClassName( - input.getControllerDescriptor().getClassName()) + "]"); + input.getControllerDescriptor().getClassName()) + "]"); } for (DAGProtos.RootInputLeafOutputProto output : vertexPlan.getOutputsList()) { Graph.Node outputNode = graph.getNode(sanitizeLabelForViz( - vertexPlan.getName()) + vertexPlan.getName()) + "_" + sanitizeLabelForViz(output.getName())); outputNode.setLabel(sanitizeLabelForViz(vertexPlan.getName()) + "[" + sanitizeLabelForViz(output.getName()) + "]"); outputNode.setShape("box"); n.addEdge(outputNode, "Output" + " [outputClass=" + getShortClassName( - output.getIODescriptor().getClassName()) + output.getIODescriptor().getClassName()) + ", committer=" + getShortClassName( - output.getControllerDescriptor().getClassName()) + "]"); + output.getControllerDescriptor().getClassName()) + "]"); } } @@ -188,11 +190,11 @@ public static void generateDAGVizFile(final DAG dag, Graph.Node n = graph.getNode(sanitizeLabelForViz( e.getInputVertexName())); n.addEdge(graph.getNode(sanitizeLabelForViz( - e.getOutputVertexName())), + e.getOutputVertexName())), "[" + "input=" + getShortClassName(e.getEdgeSource().getClassName()) + ", output=" + getShortClassName( - e.getEdgeDestination().getClassName()) + e.getEdgeDestination().getClassName()) + ", dataMovement=" + e.getDataMovementType().name().trim() + ", schedulingType=" + e.getSchedulingType().name().trim() + "]"); @@ -215,8 +217,8 @@ public static void generateDAGVizFile(final DAG dag, try { LOG.info("Generating DAG graphviz file" - + ", dagId=" + dagId.toString() - + ", filePath=" + outputFile); + + ", dagId=" + dagId.toString() + + ", filePath=" + outputFile); graph.save(outputFile); } catch (Exception e) { LOG.warn("Error occurred when trying to save graph structure" @@ -226,6 +228,7 @@ public static void generateDAGVizFile(final DAG dag, /** * Get the short name of the class. + * * @param className long name * @return short name */ @@ -239,6 +242,7 @@ private static String getShortClassName(final String className) { /** * Replace some characters with underscores. + * * @param label label to sanitize * @return the label with the replaced characters */ diff --git a/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java b/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java index d0580bbae9..6eb644bdbe 100644 --- a/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java +++ b/tez-dag/src/main/java/org/apache/tez/client/LocalClient.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -111,7 +111,6 @@ public void init(TezConfiguration tezConf) { TezConfiguration.TEZ_LOCAL_MODE_WITHOUT_NETWORK_DEFAULT); } - @Override public void start() { // LocalClients are shared between TezClient and DAGClients, which can cause stop / start / close @@ -157,7 +156,7 @@ public ApplicationId submitApplication(ApplicationSubmissionContext appContext) @Override public void killApplication(ApplicationId appId) { try { - if (clientHandler != null){ + if (clientHandler != null) { clientHandler.shutdownAM(); } } catch (TezException e) { @@ -230,23 +229,23 @@ protected FinalApplicationStatus convertDAGAppMasterStateToFinalYARNState( protected YarnApplicationState convertDAGAppMasterState(DAGAppMasterState dagAppMasterState) { switch (dagAppMasterState) { - case NEW: - return YarnApplicationState.NEW; - case INITED: - case RECOVERING: - case IDLE: - case RUNNING: - return YarnApplicationState.RUNNING; - case SUCCEEDED: - return YarnApplicationState.FINISHED; - case FAILED: - return YarnApplicationState.FAILED; - case KILLED: - return YarnApplicationState.KILLED; - case ERROR: - return YarnApplicationState.FAILED; - default: - return YarnApplicationState.SUBMITTED; + case NEW: + return YarnApplicationState.NEW; + case INITED: + case RECOVERING: + case IDLE: + case RUNNING: + return YarnApplicationState.RUNNING; + case SUCCEEDED: + return YarnApplicationState.FINISHED; + case FAILED: + return YarnApplicationState.FAILED; + case KILLED: + return YarnApplicationState.KILLED; + case ERROR: + return YarnApplicationState.FAILED; + default: + return YarnApplicationState.SUBMITTED; } } @@ -355,11 +354,11 @@ public void run() { dagAppMaster = createDAGAppMaster(applicationAttemptId, cId, currentHost, nmPort, nmHttpPort, SystemClock.getInstance(), appSubmitTime, isSession, userDir.toUri().getPath(), - new String[] {localDir.toUri().getPath()}, new String[] {logDir.toUri().getPath()}, + new String[]{localDir.toUri().getPath()}, new String[]{logDir.toUri().getPath()}, amCredentials, UserGroupInformation.getCurrentUser().getShortUserName()); DAGAppMaster.initAndStartAppMaster(dagAppMaster, conf); clientHandler = new DAGClientHandler(dagAppMaster); - ((AsyncDispatcher)dagAppMaster.getDispatcher()).setDrainEventsOnStop(); + ((AsyncDispatcher) dagAppMaster.getDispatcher()).setDrainEventsOnStop(); } catch (Throwable t) { LOG.error("Error starting DAGAppMaster", t); if (dagAppMaster != null) { @@ -394,17 +393,17 @@ protected DAGAppMaster createDAGAppMaster(ApplicationAttemptId applicationAttemp .getAmPluginDescriptor(); return isLocalWithoutNetwork - ? new LocalDAGAppMaster(applicationAttemptId, cId, currentHost, nmPort, nmHttpPort, - SystemClock.getInstance(), appSubmitTime, isSession, userDir, localDirs, logDirs, - versionInfo.getVersion(), credentials, jobUserName, amPluginDescriptorProto) - : new DAGAppMaster(applicationAttemptId, cId, currentHost, nmPort, nmHttpPort, - SystemClock.getInstance(), appSubmitTime, isSession, userDir, localDirs, logDirs, - versionInfo.getVersion(), credentials, jobUserName, amPluginDescriptorProto); + ? new LocalDAGAppMaster(applicationAttemptId, cId, currentHost, nmPort, nmHttpPort, + SystemClock.getInstance(), appSubmitTime, isSession, userDir, localDirs, logDirs, + versionInfo.getVersion(), credentials, jobUserName, amPluginDescriptorProto) + : new DAGAppMaster(applicationAttemptId, cId, currentHost, nmPort, nmHttpPort, + SystemClock.getInstance(), appSubmitTime, isSession, userDir, localDirs, logDirs, + versionInfo.getVersion(), credentials, jobUserName, amPluginDescriptorProto); } @Override public TezAppMasterStatus getAMStatus(Configuration configuration, ApplicationId appId, - UserGroupInformation ugi) throws TezException, ServiceException, IOException { + UserGroupInformation ugi) throws TezException, ServiceException, IOException { if (isLocalWithoutNetwork) { if (clientHandler == null) { return TezAppMasterStatus.INITIALIZING; @@ -416,8 +415,8 @@ public TezAppMasterStatus getAMStatus(Configuration configuration, ApplicationId @Override public DAGClient submitDag(org.apache.tez.dag.api.DAG dag, SubmitDAGRequestProto request, - String clientName, ApplicationId sessionAppId, long clientTimeout, UserGroupInformation ugi, - TezConfiguration tezConf) throws IOException, TezException, DAGSubmissionTimedOut { + String clientName, ApplicationId sessionAppId, long clientTimeout, UserGroupInformation ugi, + TezConfiguration tezConf) throws IOException, TezException, DAGSubmissionTimedOut { Map additionalResources = null; if (request.hasAdditionalAmResources()) { @@ -428,25 +427,25 @@ public DAGClient submitDag(org.apache.tez.dag.api.DAG dag, SubmitDAGRequestProto String dagId = dagAppMaster.submitDAGToAppMaster(request.getDAGPlan(), additionalResources); return isLocalWithoutNetwork - ? new DAGClientImplLocal(sessionAppId, dagId, tezConf, this, - ugi, new BiFunction, Long, DAGStatus>() { - @Override - public DAGStatus apply(Set statusOpts, Long timeout) { - try { - return clientHandler.getDAGStatus(dagId, statusOpts, timeout); - } catch (TezException e) { - throw new RuntimeException(e); - } - } - }) - : new DAGClientImpl(sessionAppId, dagId, tezConf, this, ugi); + ? new DAGClientImplLocal(sessionAppId, dagId, tezConf, this, + ugi, new BiFunction, Long, DAGStatus>() { + @Override + public DAGStatus apply(Set statusOpts, Long timeout) { + try { + return clientHandler.getDAGStatus(dagId, statusOpts, timeout); + } catch (TezException e) { + throw new RuntimeException(e); + } + } + }) + : new DAGClientImpl(sessionAppId, dagId, tezConf, this, ugi); } @Override public boolean shutdownSession(Configuration configuration, ApplicationId sessionAppId, - UserGroupInformation ugi) throws TezException, IOException, ServiceException { + UserGroupInformation ugi) throws TezException, IOException, ServiceException { if (isLocalWithoutNetwork) { - if (clientHandler != null){ + if (clientHandler != null) { clientHandler.shutdownAM(); } return true; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientHandler.java index 1de62012e7..06b86d1bcc 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientHandler.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientHandler.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -45,7 +45,7 @@ public class DAGClientHandler { private DAGAppMaster dagAppMaster; private final AtomicLong lastHeartbeatTime; - + public DAGClientHandler(DAGAppMaster dagAppMaster) { this.dagAppMaster = dagAppMaster; this.lastHeartbeatTime = new AtomicLong(dagAppMaster.getContext().getClock().getTime()); @@ -64,17 +64,17 @@ public List getAllDAGs() throws TezException { } public DAGStatus getDAGStatus(String dagIdStr, - Set statusOptions) throws TezException { + Set statusOptions) throws TezException { return getDAG(dagIdStr).getDAGStatus(statusOptions); } public DAGStatus getDAGStatus(String dagIdStr, - Set statusOptions, long timeout) throws TezException { + Set statusOptions, long timeout) throws TezException { return getDAG(dagIdStr).getDAGStatus(statusOptions, timeout); } public VertexStatus getVertexStatus(String dagIdStr, String vertexName, - Set statusOptions) throws TezException { + Set statusOptions) throws TezException { VertexStatus status = getDAG(dagIdStr).getVertexStatus(vertexName, statusOptions); if (status == null) { @@ -122,7 +122,7 @@ private String getClientInfo() throws TezException { throw new TezException(ie); } String message = callerUGI.toString(); - if(null != Server.getRemoteAddress()) { + if (null != Server.getRemoteAddress()) { message += " at " + Server.getRemoteAddress(); } return message; @@ -137,7 +137,7 @@ public void tryKillDAG(String dagIdStr) throws TezException { } public synchronized String submitDAG(DAGPlan dagPlan, - Map additionalAmResources) throws TezException { + Map additionalAmResources) throws TezException { return dagAppMaster.submitDAGToAppMaster(dagPlan, additionalAmResources); } @@ -152,19 +152,19 @@ public synchronized void shutdownAM() throws TezException { public synchronized TezAppMasterStatus getTezAppMasterStatus() throws TezException { switch (dagAppMaster.getState()) { - case NEW: - case INITED: - return TezAppMasterStatus.INITIALIZING; - case IDLE: - return TezAppMasterStatus.READY; - case RECOVERING: - case RUNNING: - return TezAppMasterStatus.RUNNING; - case ERROR: - case FAILED: - case SUCCEEDED: - case KILLED: - return TezAppMasterStatus.SHUTDOWN; + case NEW: + case INITED: + return TezAppMasterStatus.INITIALIZING; + case IDLE: + return TezAppMasterStatus.READY; + case RECOVERING: + case RUNNING: + return TezAppMasterStatus.RUNNING; + case ERROR: + case FAILED: + case SUCCEEDED: + case KILLED: + return TezAppMasterStatus.SHUTDOWN; } return TezAppMasterStatus.INITIALIZING; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientServer.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientServer.java index 204024489f..8e9901c5fd 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientServer.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientServer.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.api.client; @@ -54,7 +54,7 @@ public class DAGClientServer extends AbstractService { final FileSystem stagingFs; public DAGClientServer(DAGClientHandler realInstance, - ApplicationAttemptId attemptId, FileSystem stagingFs) { + ApplicationAttemptId attemptId, FileSystem stagingFs) { super("DAGClientRPCServer"); this.realInstance = realInstance; this.secretManager = new ClientToAMTokenSecretManager(attemptId, null); @@ -71,17 +71,17 @@ public void serviceStart() { new DAGClientAMProtocolBlockingPBServerImpl(realInstance, stagingFs); BlockingService blockingService = - DAGClientAMProtocol.newReflectiveBlockingService(service); + DAGClientAMProtocol.newReflectiveBlockingService(service); int numHandlers = conf.getInt(TezConfiguration.TEZ_AM_CLIENT_THREAD_COUNT, - TezConfiguration.TEZ_AM_CLIENT_THREAD_COUNT_DEFAULT); + TezConfiguration.TEZ_AM_CLIENT_THREAD_COUNT_DEFAULT); if (numHandlers < 2) { numHandlers = 2; } server = createServer(DAGClientAMProtocolBlockingPB.class, addr, conf, - numHandlers, blockingService, TezConfiguration.TEZ_AM_CLIENT_AM_PORT_RANGE); - + numHandlers, blockingService, TezConfiguration.TEZ_AM_CLIENT_AM_PORT_RANGE); + // Enable service authorization? if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, @@ -107,7 +107,7 @@ private void refreshServiceAcls(Configuration configuration, PolicyProvider poli @Override public void serviceStop() { - if(server != null) { + if (server != null) { server.stop(); } } @@ -115,7 +115,7 @@ public void serviceStop() { public InetSocketAddress getBindAddress() { return bindAddress; } - + public void setClientAMSecretKey(ByteBuffer key) { if (key != null && key.hasRemaining()) { // non-empty key. must be useful @@ -124,8 +124,8 @@ public void setClientAMSecretKey(ByteBuffer key) { } private Server createServer(Class pbProtocol, InetSocketAddress addr, Configuration conf, - int numHandlers, - BlockingService blockingService, String portRangeConfig) throws IOException { + int numHandlers, + BlockingService blockingService, String portRangeConfig) throws IOException { RPC.setProtocolEngine(conf, pbProtocol, ProtobufRpcEngine.class); RPC.Server server = new RPC.Builder(conf).setProtocol(pbProtocol) .setInstance(blockingService).setBindAddress(addr.getHostString()) diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGStatusBuilder.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGStatusBuilder.java index 931c6d05ca..1d5cb83dad 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGStatusBuilder.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGStatusBuilder.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.api.client; @@ -73,26 +73,26 @@ public DAGStatusProto getProto() { } private DAGStatusStateProto getProtoState(DAGState state) { - switch(state) { - case NEW: - case INITED: - return DAGStatusStateProto.DAG_INITING; - case RUNNING: - return DAGStatusStateProto.DAG_RUNNING; - case COMMITTING: - return DAGStatusStateProto.DAG_COMMITTING; - case SUCCEEDED: - return DAGStatusStateProto.DAG_SUCCEEDED; - case FAILED: - return DAGStatusStateProto.DAG_FAILED; - case KILLED: - return DAGStatusStateProto.DAG_KILLED; - case TERMINATING: - return DAGStatusStateProto.DAG_TERMINATING; - case ERROR: - return DAGStatusStateProto.DAG_ERROR; - default: - throw new TezUncheckedException("Unsupported value for DAGState : " + state); + switch (state) { + case NEW: + case INITED: + return DAGStatusStateProto.DAG_INITING; + case RUNNING: + return DAGStatusStateProto.DAG_RUNNING; + case COMMITTING: + return DAGStatusStateProto.DAG_COMMITTING; + case SUCCEEDED: + return DAGStatusStateProto.DAG_SUCCEEDED; + case FAILED: + return DAGStatusStateProto.DAG_FAILED; + case KILLED: + return DAGStatusStateProto.DAG_KILLED; + case TERMINATING: + return DAGStatusStateProto.DAG_TERMINATING; + case ERROR: + return DAGStatusStateProto.DAG_ERROR; + default: + throw new TezUncheckedException("Unsupported value for DAGState : " + state); } } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/ProgressBuilder.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/ProgressBuilder.java index 9dc13549cb..7d807658b3 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/api/client/ProgressBuilder.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/ProgressBuilder.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.api.client; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/VertexStatusBuilder.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/VertexStatusBuilder.java index 0304fc9d6f..31fdcf5c78 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/api/client/VertexStatusBuilder.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/VertexStatusBuilder.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.api.client; @@ -56,7 +56,7 @@ public void setProgress(ProgressBuilder progress) { public void setVertexCounters(TezCounters counters) { getBuilder().setVertexCounters( - DagTypeConverters.convertTezCountersToProto(counters)); + DagTypeConverters.convertTezCountersToProto(counters)); } public VertexStatusProto getProto() { @@ -65,7 +65,7 @@ public VertexStatusProto getProto() { @VisibleForTesting static VertexStatusStateProto getProtoState(VertexState state) { - switch(state) { + switch (state) { case NEW: return VertexStatusStateProto.VERTEX_NEW; case INITIALIZING: diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPBServerImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPBServerImpl.java index 5c24a27908..5760c99447 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPBServerImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPBServerImpl.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.api.client.rpc; @@ -78,23 +78,23 @@ private UserGroupInformation getRPCUser() throws ServiceException { @Override public GetAllDAGsResponseProto getAllDAGs(RpcController controller, - GetAllDAGsRequestProto request) throws ServiceException { + GetAllDAGsRequestProto request) throws ServiceException { UserGroupInformation user = getRPCUser(); if (!real.getACLManager().checkAMViewAccess(user)) { throw new AccessControlException("User " + user + " cannot perform AM view operation"); } real.updateLastHeartbeatTime(); - try{ + try { List dagIds = real.getAllDAGs(); return GetAllDAGsResponseProto.newBuilder().addAllDagId(dagIds).build(); - } catch(TezException e) { + } catch (TezException e) { throw wrapException(e); } } @Override public GetDAGStatusResponseProto getDAGStatus(RpcController controller, - GetDAGStatusRequestProto request) throws ServiceException { + GetDAGStatusRequestProto request) throws ServiceException { UserGroupInformation user = getRPCUser(); try { String dagId = request.getDagId(); @@ -105,12 +105,12 @@ public GetDAGStatusResponseProto getDAGStatus(RpcController controller, real.updateLastHeartbeatTime(); DAGStatus status; status = real.getDAGStatus(dagId, - DagTypeConverters.convertStatusGetOptsFromProto( - request.getStatusOptionsList()), timeout); + DagTypeConverters.convertStatusGetOptsFromProto( + request.getStatusOptionsList()), timeout); assert status instanceof DAGStatusBuilder; DAGStatusBuilder builder = (DAGStatusBuilder) status; return GetDAGStatusResponseProto.newBuilder(). - setDagStatus(builder.getProto()).build(); + setDagStatus(builder.getProto()).build(); } catch (TezException e) { throw wrapException(e); } @@ -118,7 +118,7 @@ public GetDAGStatusResponseProto getDAGStatus(RpcController controller, @Override public GetVertexStatusResponseProto getVertexStatus(RpcController controller, - GetVertexStatusRequestProto request) throws ServiceException { + GetVertexStatusRequestProto request) throws ServiceException { UserGroupInformation user = getRPCUser(); try { String dagId = request.getDagId(); @@ -128,12 +128,12 @@ public GetVertexStatusResponseProto getVertexStatus(RpcController controller, real.updateLastHeartbeatTime(); String vertexName = request.getVertexName(); VertexStatus status = real.getVertexStatus(dagId, vertexName, - DagTypeConverters.convertStatusGetOptsFromProto( - request.getStatusOptionsList())); + DagTypeConverters.convertStatusGetOptsFromProto( + request.getStatusOptionsList())); assert status instanceof VertexStatusBuilder; VertexStatusBuilder builder = (VertexStatusBuilder) status; return GetVertexStatusResponseProto.newBuilder(). - setVertexStatus(builder.getProto()).build(); + setVertexStatus(builder.getProto()).build(); } catch (TezException e) { throw wrapException(e); } @@ -141,7 +141,7 @@ public GetVertexStatusResponseProto getVertexStatus(RpcController controller, @Override public TryKillDAGResponseProto tryKillDAG(RpcController controller, - TryKillDAGRequestProto request) throws ServiceException { + TryKillDAGRequestProto request) throws ServiceException { UserGroupInformation user = getRPCUser(); try { String dagId = request.getDagId(); @@ -158,13 +158,13 @@ public TryKillDAGResponseProto tryKillDAG(RpcController controller, @Override public SubmitDAGResponseProto submitDAG(RpcController controller, - SubmitDAGRequestProto request) throws ServiceException { + SubmitDAGRequestProto request) throws ServiceException { UserGroupInformation user = getRPCUser(); if (!real.getACLManager().checkAMModifyAccess(user)) { throw new AccessControlException("User " + user + " cannot perform AM modify operation"); } real.updateLastHeartbeatTime(); - try{ + try { if (request.hasSerializedRequestPath()) { // need to deserialize large request from hdfs Path requestPath = new Path(request.getSerializedRequestPath()); @@ -186,18 +186,18 @@ public SubmitDAGResponseProto submitDAG(RpcController controller, } String dagId = real.submitDAG(dagPlan, additionalResources); return SubmitDAGResponseProto.newBuilder().setDagId(dagId).build(); - } catch(IOException | TezException e) { + } catch (IOException | TezException e) { throw wrapException(e); } } - ServiceException wrapException(Exception e){ + ServiceException wrapException(Exception e) { return new ServiceException(e); } @Override public ShutdownSessionResponseProto shutdownSession(RpcController arg0, - ShutdownSessionRequestProto arg1) throws ServiceException { + ShutdownSessionRequestProto arg1) throws ServiceException { UserGroupInformation user = getRPCUser(); if (!real.getACLManager().checkAMModifyAccess(user)) { throw new AccessControlException("User " + user + " cannot perform AM modify operation"); @@ -206,14 +206,14 @@ public ShutdownSessionResponseProto shutdownSession(RpcController arg0, try { real.shutdownAM(); return ShutdownSessionResponseProto.newBuilder().build(); - } catch(TezException e) { + } catch (TezException e) { throw wrapException(e); } } @Override public GetAMStatusResponseProto getAMStatus(RpcController controller, - GetAMStatusRequestProto request) throws ServiceException { + GetAMStatusRequestProto request) throws ServiceException { UserGroupInformation user = getRPCUser(); if (!real.getACLManager().checkAMViewAccess(user)) { throw new AccessControlException("User " + user + " cannot perform AM view operation"); @@ -222,9 +222,9 @@ public GetAMStatusResponseProto getAMStatus(RpcController controller, try { TezAppMasterStatus sessionStatus = real.getTezAppMasterStatus(); return GetAMStatusResponseProto.newBuilder().setStatus( - DagTypeConverters.convertTezAppMasterStatusToProto(sessionStatus)) + DagTypeConverters.convertTezAppMasterStatusToProto(sessionStatus)) .build(); - } catch(TezException e) { + } catch (TezException e) { throw wrapException(e); } } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java index 6cb3df0b5a..f7102fed6c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/AMInfo.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,16 +23,26 @@ public interface AMInfo { public ApplicationAttemptId getAppAttemptId(); + public long getStartTime(); + public ContainerId getContainerId(); + public String getNodeManagerHost(); + public int getNodeManagerPort(); + public int getNodeManagerHttpPort(); public void setAppAttemptId(ApplicationAttemptId appAttemptId); + public void setStartTime(long startTime); + public void setContainerId(ContainerId containerId); + public void setNodeManagerHost(String nmHost); + public void setNodeManagerPort(int nmPort); + public void setNodeManagerHttpPort(int mnHttpPort); } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptReport.java b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptReport.java index a2ee7e38ac..fe114fde25 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptReport.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptReport.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,41 +24,66 @@ public interface TaskAttemptReport { public abstract TezTaskAttemptID getTaskAttemptId(); + public abstract TaskAttemptState getTaskAttemptState(); + public abstract float getProgress(); + public abstract long getStartTime(); + public abstract long getFinishTime(); + /** @return the shuffle finish time. Applicable only for reduce attempts */ public abstract long getShuffleFinishTime(); + /** @return the sort/merge finish time. Applicable only for reduce attempts */ public abstract long getSortFinishTime(); + public abstract TezCounters getCounters(); + public abstract String getDiagnosticInfo(); + public abstract String getStateString(); + public abstract String getNodeManagerHost(); + public abstract int getNodeManagerPort(); + public abstract int getNodeManagerHttpPort(); + public abstract ContainerId getContainerId(); public abstract void setTaskAttemptId(TezTaskAttemptID taskAttemptId); + public abstract void setTaskAttemptState(TaskAttemptState taskAttemptState); + public abstract void setProgress(float progress); + public abstract void setStartTime(long startTime); + public abstract void setFinishTime(long finishTime); + public abstract void setCounters(TezCounters counters); + public abstract void setDiagnosticInfo(String diagnosticInfo); + public abstract void setStateString(String stateString); + public abstract void setNodeManagerHost(String nmHost); + public abstract void setNodeManagerPort(int nmPort); + public abstract void setNodeManagerHttpPort(int nmHttpPort); + public abstract void setContainerId(ContainerId containerId); - - /** + + /** * Set the shuffle finish time. Applicable only for reduce attempts * @param time the time the shuffle finished. */ public abstract void setShuffleFinishTime(long time); - /** + + /** * Set the sort/merge finish time. Applicable only for reduce attempts * @param time the time the shuffle finished. */ diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptState.java b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptState.java index 926835a97d..65fb107345 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptState.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskAttemptState.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,9 +19,9 @@ package org.apache.tez.dag.api.oldrecords; public enum TaskAttemptState { - NEW, - STARTING, - RUNNING, + NEW, + STARTING, + RUNNING, SUCCEEDED, FAILED, KILLED diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskReport.java b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskReport.java index 4d2ac62321..197cb5c490 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskReport.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskReport.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,15 +22,22 @@ public interface TaskReport { public abstract TezTaskID getTaskId(); + public abstract TaskState getTaskState(); + public abstract float getProgress(); + public abstract long getStartTime(); + public abstract long getFinishTime(); public abstract void setTaskId(TezTaskID taskId); + public abstract void setTaskState(TaskState taskState); + public abstract void setProgress(float progress); + public abstract void setStartTime(long startTime); - public abstract void setFinishTime(long finishTime); + public abstract void setFinishTime(long finishTime); } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskState.java b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskState.java index c6b87673aa..f9adc96e44 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskState.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/api/oldrecords/TaskState.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java index c9a7083c1d..56b0da9dc0 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app; @@ -43,7 +43,6 @@ import com.google.common.util.concurrent.ListeningExecutorService; - /** * Context interface for sharing information across components in Tez DAG */ @@ -57,9 +56,9 @@ public interface AppContext { ApplicationId getApplicationID(); TezDAGID getCurrentDAGID(); - + long getCumulativeCPUTime(); - + long getCumulativeGCTime(); ApplicationAttemptId getApplicationAttemptId(); @@ -127,15 +126,21 @@ public interface AppContext { Credentials getAppCredentials(); public Integer getTaskCommunicatorIdentifier(String name); + public Integer getTaskScheduerIdentifier(String name); + public Integer getContainerLauncherIdentifier(String name); public String getTaskCommunicatorName(int taskCommId); + public String getTaskSchedulerName(int schedulerId); + public String getContainerLauncherName(int launcherId); public String getTaskCommunicatorClassName(int taskCommId); + public String getTaskSchedulerClassName(int schedulerId); + public String getContainerLauncherClassName(int launcherId); public HadoopShim getHadoopShim(); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/ClusterInfo.java b/tez-dag/src/main/java/org/apache/tez/dag/app/ClusterInfo.java index b30783887a..d7452533f8 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/ClusterInfo.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/ClusterInfo.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -27,7 +27,7 @@ public class ClusterInfo { public ClusterInfo() { } - + public ClusterInfo(Resource maxCapability) { this.maxContainerCapability = maxCapability; initialized = true; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerContext.java b/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerContext.java index f2c7d5f700..459e60a8f0 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerContext.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerContext.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -47,7 +47,7 @@ public class ContainerContext { // FIXME Add support for service meta data comparisons public ContainerContext(Map localResources, - Credentials credentials, Map environment, String javaOpts) { + Credentials credentials, Map environment, String javaOpts) { Objects.requireNonNull(localResources, "localResources should not be null"); Objects.requireNonNull(credentials, "credentials should not be null"); @@ -59,10 +59,10 @@ public ContainerContext(Map localResources, this.javaOpts = javaOpts; this.vertex = null; } - + public ContainerContext(Map localResources, - Credentials credentials, Map environment, String javaOpts, - @Nullable Vertex vertex) { + Credentials credentials, Map environment, String javaOpts, + @Nullable Vertex vertex) { Objects.requireNonNull(localResources, "localResources should not be null"); Objects.requireNonNull(credentials, "credentials should not be null"); @@ -105,8 +105,8 @@ public boolean isSuperSet(ContainerContext otherContext) { if (!this.javaOpts.equals(otherContext.javaOpts)) { if (LOG.isDebugEnabled()) { LOG.debug("Incompatible java opts, " - + ", this=" + this.javaOpts - + ", other=" + otherContext.javaOpts); + + ", this=" + this.javaOpts + + ", other=" + otherContext.javaOpts); } return false; } @@ -114,7 +114,7 @@ public boolean isSuperSet(ContainerContext otherContext) { return isSuperSet(this.environment, otherContext.getEnvironment(), "Environment") && localResourcesCompatible(this.localResources, otherContext.getLocalResources()); } - + /** * @return true if this ContainerContext is an exact match of the specified * container context. @@ -127,7 +127,7 @@ public boolean isExactMatch(ContainerContext otherContext) { // YarnTezDAGChild knows how to handle the additional types in terms of // classpath modification private static boolean localResourcesCompatible(Map srcLRs, - Map reqLRs) { + Map reqLRs) { for (Entry reqLREntry : reqLRs.entrySet()) { LocalResource requestedLocalResource = srcLRs.get(reqLREntry.getKey()); if (requestedLocalResource == null) { @@ -140,7 +140,7 @@ private static boolean localResourcesCompatible(Map srcLR } return false; } - } else if(!reqLREntry.getValue().equals(requestedLocalResource)) { + } else if (!reqLREntry.getValue().equals(requestedLocalResource)) { if (LOG.isDebugEnabled()) { LOG.debug("Cannot match container: Attempting to use same target resource name: " + reqLREntry.getKey() @@ -154,7 +154,7 @@ private static boolean localResourcesCompatible(Map srcLR } private static boolean isSuperSet(Map srcMap, Map matchMap, - String matchInfo) { + String matchInfo) { for (Entry oEntry : matchMap.entrySet()) { K oKey = oEntry.getKey(); V oVal = oEntry.getValue(); @@ -230,11 +230,10 @@ public String toString() { } } sb.append("], javaOpts: ") - .append(javaOpts) - .append(", vertex: ") - .append(( vertex == null ? "null" : vertex.getLogIdentifier())); + .append(javaOpts) + .append(", vertex: ") + .append((vertex == null ? "null" : vertex.getLogIdentifier())); return sb.toString(); } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerHeartbeatHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerHeartbeatHandler.java index e494fa6069..fcd1948197 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerHeartbeatHandler.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerHeartbeatHandler.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app; @@ -27,9 +27,8 @@ public class ContainerHeartbeatHandler extends HeartbeatHandlerBase { - - public ContainerHeartbeatHandler(AppContext context, - int expectedConcurrency) { + public ContainerHeartbeatHandler(AppContext context, + int expectedConcurrency) { super(context, expectedConcurrency, "ContainerHeartbeatHandler"); } @@ -47,7 +46,6 @@ protected int getConfiguredTimeoutCheckInterval(Configuration conf) { @Override protected boolean hasTimedOut(ReportTime report, long currentTime) { return (timeOut > 0) && (currentTime > report.getLastPing() + timeOut); - } @SuppressWarnings("unchecked") @@ -56,5 +54,4 @@ protected void handleTimeOut(ContainerId containerId) { eventHandler.handle(new AMContainerEvent(containerId, AMContainerEventType.C_TIMED_OUT)); } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerLauncherContextImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerLauncherContextImpl.java index 694de075d0..60e218e916 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerLauncherContextImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerLauncherContextImpl.java @@ -71,7 +71,6 @@ public void containerLaunched(ContainerId containerId) { containerId, context.getClock().getTime(), context.getApplicationAttemptId()); context.getHistoryHandler().handle(new DAGHistoryEvent( null, lEvt)); - } @Override @@ -144,6 +143,4 @@ public void reportError(ServicePluginError servicePluginError, String message, D Objects.requireNonNull(servicePluginError, "ServiceError must be specified"); containerLauncherManager.reportError(containerLauncherIndex, servicePluginError, message, dagInfo); } - - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java index ca50b2ac6b..0e3ccfa977 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java @@ -1,25 +1,23 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app; - - import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; @@ -338,10 +336,10 @@ public class DAGAppMaster extends AbstractService { new LinkedHashMap(); public DAGAppMaster(ApplicationAttemptId applicationAttemptId, - ContainerId containerId, String nmHost, int nmPort, int nmHttpPort, - Clock clock, long appSubmitTime, boolean isSession, String workingDirectory, - String [] localDirs, String[] logDirs, String clientVersion, - Credentials credentials, String jobUserName, AMPluginDescriptorProto pluginDescriptorProto) { + ContainerId containerId, String nmHost, int nmPort, int nmHttpPort, + Clock clock, long appSubmitTime, boolean isSession, String workingDirectory, + String[] localDirs, String[] logDirs, String clientVersion, + Credentials credentials, String jobUserName, AMPluginDescriptorProto pluginDescriptorProto) { super(DAGAppMaster.class.getName()); this.clock = clock; this.startTime = clock.getTime(); @@ -375,7 +373,7 @@ public DAGAppMaster(ApplicationAttemptId applicationAttemptId, // Pull this WebAppUtils function into Tez until YARN-4186 private static String getRunningLogURL(String nodeHttpAddress, - String containerId, String user) { + String containerId, String user) { if (containerId.isEmpty() || user == null | user.isEmpty()) { return null; } @@ -443,7 +441,6 @@ protected void serviceInit(final Configuration conf) throws Exception { containerLaunchers, taskCommunicatorDescriptors, taskCommunicators, amPluginDescriptorProto, isLocal, defaultPayload); - LOG.info(buildPluginComponentLog(taskSchedulerDescriptors, taskSchedulers, "TaskSchedulers")); LOG.info(buildPluginComponentLog(containerLauncherDescriptors, containerLaunchers, "ContainerLaunchers")); LOG.info(buildPluginComponentLog(taskCommunicatorDescriptors, taskCommunicators, "TaskCommunicators")); @@ -474,9 +471,9 @@ protected void serviceInit(final Configuration conf) throws Exception { dispatcher = createDispatcher(); if (isLocal) { - conf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false); - conf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, - TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS_DEFAULT); + conf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false); + conf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, + TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS_DEFAULT); } else { dispatcher.enableExitOnDispatchException(); } @@ -521,8 +518,6 @@ protected void serviceInit(final Configuration conf) throws Exception { jobTokenSecretManager.addTokenForJob( appAttemptID.getApplicationId().toString(), sessionToken); - - //service to handle requests to TaskUmbilicalProtocol taskCommunicatorManager = createTaskCommunicatorManager(context, taskHeartbeatHandler, containerHeartbeatHandler, taskCommunicatorDescriptors); @@ -598,9 +593,9 @@ protected void serviceInit(final Configuration conf) throws Exception { if (!versionMismatch) { if (isSession) { try (BufferedInputStream sessionResourcesStream = - new BufferedInputStream( - new FileInputStream(new File(workingDirectory, - TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME)))) { + new BufferedInputStream( + new FileInputStream(new File(workingDirectory, + TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME)))) { PlanLocalResourcesProto amLocalResourceProto = PlanLocalResourcesProto .parseDelimitedFrom(sessionResourcesStream); amResources.putAll(DagTypeConverters @@ -610,7 +605,7 @@ protected void serviceInit(final Configuration conf) throws Exception { } int threadCount = conf.getInt(TezConfiguration.TEZ_AM_DAG_APPCONTEXT_THREAD_COUNT_LIMIT, - TezConfiguration.TEZ_AM_DAG_APPCONTEXT_THREAD_COUNT_LIMIT_DEFAULT); + TezConfiguration.TEZ_AM_DAG_APPCONTEXT_THREAD_COUNT_LIMIT_DEFAULT); // NOTE: LinkedBlockingQueue does not have a capacity Limit and can thus // occupy large memory chunks when numerous Runables are pending for execution ExecutorService rawExecutor = @@ -715,167 +710,167 @@ private void handleInternalError(String errDiagnosticsPrefix, String errDiagDagE protected synchronized void handle(DAGAppMasterEvent event) { String errDiagnostics; switch (event.getType()) { - case SCHEDULING_SERVICE_ERROR: - // Scheduling error - probably an issue with the communication with the RM - // In this scenario, the AM should shutdown. Expectation is that the RM - // will restart a new AM attempt. - // Should not kill the current running DAG to ensure that on restart, we - // can recover it and continue. - DAGAppMasterEventSchedulingServiceError schedulingServiceErrorEvent = - (DAGAppMasterEventSchedulingServiceError) event; - state = DAGAppMasterState.ERROR; - errDiagnostics = "Error in the TaskScheduler. Shutting down. "; - addDiagnostic(errDiagnostics - + "Error=" + schedulingServiceErrorEvent.getDiagnosticInfo()); - LOG.error(errDiagnostics); - shutdownHandler.shutdown(); - break; - case TASK_COMMUNICATOR_SERVICE_FATAL_ERROR: - case CONTAINER_LAUNCHER_SERVICE_FATAL_ERROR: - case TASK_SCHEDULER_SERVICE_FATAL_ERROR: - // A fatal error from the pluggable services. The AM cannot continue operation, and should - // be shutdown. The AM should not be restarted for recovery. - DAGAppMasterEventUserServiceFatalError usfe = (DAGAppMasterEventUserServiceFatalError) event; - Throwable error = usfe.getError(); - errDiagnostics = "Service Error: " + usfe.getDiagnosticInfo() - + ", eventType=" + event.getType() - + ", exception=" + (usfe.getError() == null ? "None" : ExceptionUtils.getStackTrace(usfe.getError())); - LOG.error(errDiagnostics, error); - addDiagnostic(errDiagnostics); - - handleInternalError("Service error: " + event.getType(), errDiagnostics); - break; - case INTERNAL_ERROR: - handleInternalError("DAGAppMaster Internal Error occurred", - "DAGAppMaster Internal Error occurred"); - break; - case DAG_FINISHED: - DAGAppMasterEventDAGFinished finishEvt = - (DAGAppMasterEventDAGFinished) event; - String timeStamp = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(Calendar.getInstance().getTime()); - System.err.println(timeStamp + " Completed Dag: " + finishEvt.getDAGId()); - System.out.println(timeStamp + " Completed Dag: " + finishEvt.getDAGId()); - // Stop vertex services if any - stopVertexServices(currentDAG); - if (!isSession) { - LOG.info("Not a session, AM will unregister as DAG has completed"); - this.taskSchedulerManager.setShouldUnregisterFlag(); - _updateLoggers(currentDAG, "_post"); - setStateOnDAGCompletion(); - LOG.info("Shutting down on completion of dag:" + finishEvt.getDAGId()); + case SCHEDULING_SERVICE_ERROR: + // Scheduling error - probably an issue with the communication with the RM + // In this scenario, the AM should shutdown. Expectation is that the RM + // will restart a new AM attempt. + // Should not kill the current running DAG to ensure that on restart, we + // can recover it and continue. + DAGAppMasterEventSchedulingServiceError schedulingServiceErrorEvent = + (DAGAppMasterEventSchedulingServiceError) event; + state = DAGAppMasterState.ERROR; + errDiagnostics = "Error in the TaskScheduler. Shutting down. "; + addDiagnostic(errDiagnostics + + "Error=" + schedulingServiceErrorEvent.getDiagnosticInfo()); + LOG.error(errDiagnostics); shutdownHandler.shutdown(); - } else { - LOG.info("DAG completed, dagId=" + finishEvt.getDAGId() + ", dagState=" - + finishEvt.getDAGState()); - lastDAGCompletionTime = clock.getTime(); - _updateLoggers(currentDAG, "_post"); - if (this.historyEventHandler.hasRecoveryFailed()) { - String recoveryErrorMsg = "Recovery had a fatal error, shutting down session after" + - " DAG completion"; - LOG.warn(recoveryErrorMsg); - addDiagnostic(recoveryErrorMsg); - sessionStopped.set(true); - } - switch(finishEvt.getDAGState()) { - case SUCCEEDED: - if (!currentDAG.getName().startsWith( - TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX)) { - successfulDAGs.incrementAndGet(); - } - break; - case FAILED: - if (!currentDAG.getName().startsWith( - TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX)) { - failedDAGs.incrementAndGet(); - } - break; - case KILLED: - if (!currentDAG.getName().startsWith( - TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX)) { - killedDAGs.incrementAndGet(); - } - break; - case ERROR: - if (!currentDAG.getName().startsWith( - TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX)) { - failedDAGs.incrementAndGet(); - } - // This is a pass-through. Kill the AM if DAG state is ERROR. - default: - LOG.error("Received a DAG Finished Event with state=" - + finishEvt.getDAGState() - + ". Error. Shutting down."); - addDiagnostic("DAG completed with an ERROR state. Shutting down AM"); - state = DAGAppMasterState.ERROR; + break; + case TASK_COMMUNICATOR_SERVICE_FATAL_ERROR: + case CONTAINER_LAUNCHER_SERVICE_FATAL_ERROR: + case TASK_SCHEDULER_SERVICE_FATAL_ERROR: + // A fatal error from the pluggable services. The AM cannot continue operation, and should + // be shutdown. The AM should not be restarted for recovery. + DAGAppMasterEventUserServiceFatalError usfe = (DAGAppMasterEventUserServiceFatalError) event; + Throwable error = usfe.getError(); + errDiagnostics = "Service Error: " + usfe.getDiagnosticInfo() + + ", eventType=" + event.getType() + + ", exception=" + (usfe.getError() == null ? "None" : ExceptionUtils.getStackTrace(usfe.getError())); + LOG.error(errDiagnostics, error); + addDiagnostic(errDiagnostics); + + handleInternalError("Service error: " + event.getType(), errDiagnostics); + break; + case INTERNAL_ERROR: + handleInternalError("DAGAppMaster Internal Error occurred", + "DAGAppMaster Internal Error occurred"); + break; + case DAG_FINISHED: + DAGAppMasterEventDAGFinished finishEvt = + (DAGAppMasterEventDAGFinished) event; + String timeStamp = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(Calendar.getInstance().getTime()); + System.err.println(timeStamp + " Completed Dag: " + finishEvt.getDAGId()); + System.out.println(timeStamp + " Completed Dag: " + finishEvt.getDAGId()); + // Stop vertex services if any + stopVertexServices(currentDAG); + if (!isSession) { + LOG.info("Not a session, AM will unregister as DAG has completed"); this.taskSchedulerManager.setShouldUnregisterFlag(); + _updateLoggers(currentDAG, "_post"); + setStateOnDAGCompletion(); + LOG.info("Shutting down on completion of dag:" + finishEvt.getDAGId()); shutdownHandler.shutdown(); - break; - } - if (!state.equals(DAGAppMasterState.ERROR)) { - if (!sessionStopped.get()) { - LOG.info("Central Dispatcher queue size after DAG completion, before cleanup: " + - dispatcher.getQueueSize()); - LOG.info("Waiting for next DAG to be submitted."); - - // Sending this via the event queue, in case there are pending events which need to be - // processed. TaskKilled for example, or ContainerCompletions. - // The DAG needs to be part of the event, since the dag can get reset when the next - // dag is submitted. The next DAG, however, will not start executing till the cleanup - // is complete, since execution start is on the same dispatcher. - sendEvent(new DAGAppMasterEventDagCleanup(context.getCurrentDAG())); - - // Leaving the taskSchedulerEventHandler here for now. Doesn't generate new events. - // However, eventually it needs to be moved out. - this.taskSchedulerManager.dagCompleted(); - } else { - LOG.info("Session shutting down now."); - this.taskSchedulerManager.setShouldUnregisterFlag(); - if (this.historyEventHandler.hasRecoveryFailed()) { - state = DAGAppMasterState.FAILED; + } else { + LOG.info("DAG completed, dagId=" + finishEvt.getDAGId() + ", dagState=" + + finishEvt.getDAGState()); + lastDAGCompletionTime = clock.getTime(); + _updateLoggers(currentDAG, "_post"); + if (this.historyEventHandler.hasRecoveryFailed()) { + String recoveryErrorMsg = "Recovery had a fatal error, shutting down session after" + + " DAG completion"; + LOG.warn(recoveryErrorMsg); + addDiagnostic(recoveryErrorMsg); + sessionStopped.set(true); + } + switch (finishEvt.getDAGState()) { + case SUCCEEDED: + if (!currentDAG.getName().startsWith( + TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX)) { + successfulDAGs.incrementAndGet(); + } + break; + case FAILED: + if (!currentDAG.getName().startsWith( + TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX)) { + failedDAGs.incrementAndGet(); + } + break; + case KILLED: + if (!currentDAG.getName().startsWith( + TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX)) { + killedDAGs.incrementAndGet(); + } + break; + case ERROR: + if (!currentDAG.getName().startsWith( + TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX)) { + failedDAGs.incrementAndGet(); + } + // This is a pass-through. Kill the AM if DAG state is ERROR. + default: + LOG.error("Received a DAG Finished Event with state=" + + finishEvt.getDAGState() + + ". Error. Shutting down."); + addDiagnostic("DAG completed with an ERROR state. Shutting down AM"); + state = DAGAppMasterState.ERROR; + this.taskSchedulerManager.setShouldUnregisterFlag(); + shutdownHandler.shutdown(); + break; + } + if (!state.equals(DAGAppMasterState.ERROR)) { + if (!sessionStopped.get()) { + LOG.info("Central Dispatcher queue size after DAG completion, before cleanup: " + + dispatcher.getQueueSize()); + LOG.info("Waiting for next DAG to be submitted."); + + // Sending this via the event queue, in case there are pending events which need to be + // processed. TaskKilled for example, or ContainerCompletions. + // The DAG needs to be part of the event, since the dag can get reset when the next + // dag is submitted. The next DAG, however, will not start executing till the cleanup + // is complete, since execution start is on the same dispatcher. + sendEvent(new DAGAppMasterEventDagCleanup(context.getCurrentDAG())); + + // Leaving the taskSchedulerEventHandler here for now. Doesn't generate new events. + // However, eventually it needs to be moved out. + this.taskSchedulerManager.dagCompleted(); } else { - state = DAGAppMasterState.SUCCEEDED; + LOG.info("Session shutting down now."); + this.taskSchedulerManager.setShouldUnregisterFlag(); + if (this.historyEventHandler.hasRecoveryFailed()) { + state = DAGAppMasterState.FAILED; + } else { + state = DAGAppMasterState.SUCCEEDED; + } + shutdownHandler.shutdown(); } - shutdownHandler.shutdown(); } } - } - //close all fs related caches - try { - FileSystem.closeAllForUGI(context.getCurrentDAG().getDagUGI()); - } catch (IOException e) { - LOG.warn("Error occurred when trying to close FileSystem for userName " + context - .getCurrentDAG().getDagUGI().getUserName(), e); - } - break; - case AM_REBOOT: - LOG.info("Received an AM_REBOOT signal"); - this.state = DAGAppMasterState.KILLED; - shutdownHandler.shutdown(true); - break; - case DAG_CLEANUP: - DAGAppMasterEventDagCleanup cleanupEvent = (DAGAppMasterEventDagCleanup) event; - LOG.info("Cleaning up DAG: name=" + cleanupEvent.getDag().getName() + ", with id=" + - cleanupEvent.getDag().getID()); - containerLauncherManager.dagComplete(cleanupEvent.getDag().getID(), jobTokenSecretManager); - taskCommunicatorManager.dagComplete(cleanupEvent.getDag()); - nodes.dagComplete(cleanupEvent.getDag()); - containers.dagComplete(cleanupEvent.getDag()); - LOG.info("Completed cleanup for DAG: name=" + cleanupEvent.getDag().getName() + ", with id=" + - cleanupEvent.getDag().getID()); - synchronized (idleStateLock) { - state = DAGAppMasterState.IDLE; - idleStateLock.notify(); - } - break; - case NEW_DAG_SUBMITTED: - // Inform sub-components that a new DAG has been submitted. - taskSchedulerManager.dagSubmitted(); - containerLauncherManager.dagSubmitted(); - taskCommunicatorManager.dagSubmitted(); - break; - default: - throw new TezUncheckedException( - "AppMaster: No handler for event type: " + event.getType()); + //close all fs related caches + try { + FileSystem.closeAllForUGI(context.getCurrentDAG().getDagUGI()); + } catch (IOException e) { + LOG.warn("Error occurred when trying to close FileSystem for userName " + context + .getCurrentDAG().getDagUGI().getUserName(), e); + } + break; + case AM_REBOOT: + LOG.info("Received an AM_REBOOT signal"); + this.state = DAGAppMasterState.KILLED; + shutdownHandler.shutdown(true); + break; + case DAG_CLEANUP: + DAGAppMasterEventDagCleanup cleanupEvent = (DAGAppMasterEventDagCleanup) event; + LOG.info("Cleaning up DAG: name=" + cleanupEvent.getDag().getName() + ", with id=" + + cleanupEvent.getDag().getID()); + containerLauncherManager.dagComplete(cleanupEvent.getDag().getID(), jobTokenSecretManager); + taskCommunicatorManager.dagComplete(cleanupEvent.getDag()); + nodes.dagComplete(cleanupEvent.getDag()); + containers.dagComplete(cleanupEvent.getDag()); + LOG.info("Completed cleanup for DAG: name=" + cleanupEvent.getDag().getName() + ", with id=" + + cleanupEvent.getDag().getID()); + synchronized (idleStateLock) { + state = DAGAppMasterState.IDLE; + idleStateLock.notify(); + } + break; + case NEW_DAG_SUBMITTED: + // Inform sub-components that a new DAG has been submitted. + taskSchedulerManager.dagSubmitted(); + containerLauncherManager.dagSubmitted(); + taskCommunicatorManager.dagSubmitted(); + break; + default: + throw new TezUncheckedException( + "AppMaster: No handler for event type: " + event.getType()); } } @@ -883,7 +878,7 @@ private void _updateLoggers(DAG dag, String appender) { try { TezUtilsInternal.updateLoggers(dag.getID().toString() + appender); } catch (FileNotFoundException e) { - LOG.warn("Unable to update the logger. Continue with the old logger", e ); + LOG.warn("Unable to update the logger. Continue with the old logger", e); } } @@ -904,7 +899,7 @@ public void handle(DAGAppMasterEvent event) { // otherwise there may be dead-lock happen. TEZ-2204 if (DAGAppMaster.this.getServiceState() == STATE.STOPPED) { LOG.info("ignore event when DAGAppMaster is in the state of STOPPED, eventType=" - + event.getType()); + + event.getType()); return; } DAGAppMaster.this.handle(event); @@ -925,7 +920,7 @@ public void shutdown() { public void shutdown(boolean now) { LOG.info("DAGAppMasterShutdownHandler invoked"); - if(!shutdownHandled.compareAndSet(false, true)) { + if (!shutdownHandled.compareAndSet(false, true)) { LOG.info("Ignoring multiple shutdown events"); return; } @@ -1062,7 +1057,7 @@ protected void addIfService(Object object, boolean addDispatcher) { Service service = (Service) object; ServiceWithDependency sd = new ServiceWithDependency(service); services.put(service, sd); - if(addDispatcher) { + if (addDispatcher) { addIfServiceDependency(service, dispatcher); } } @@ -1082,14 +1077,14 @@ protected TaskCommunicatorManagerInterface createTaskCommunicatorManager(AppCont TaskHeartbeatHandler thh, ContainerHeartbeatHandler chh, List entityDescriptors) - throws TezException { + throws TezException { TaskCommunicatorManagerInterface tcm = new TaskCommunicatorManager(context, thh, chh, entityDescriptors); return tcm; } protected TaskHeartbeatHandler createTaskHeartbeatHandler(AppContext context, - Configuration conf) { + Configuration conf) { TaskHeartbeatHandler thh = new TaskHeartbeatHandler(context, conf.getInt( TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT, TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT_DEFAULT)); @@ -1180,7 +1175,7 @@ public List getDiagnostics() { } if (!isSession) { - if(currentDAG != null) { + if (currentDAG != null) { diagResult.addAll(currentDAG.getDiagnostics()); } } else { @@ -1197,7 +1192,7 @@ public float getProgress() { if (isSession && getState().equals(DAGAppMasterState.IDLE)) { return 0.0f; } - if(currentDAG != null) { + if (currentDAG != null) { DAGState state = currentDAG.getState(); switch (state) { case NEW: @@ -1218,25 +1213,25 @@ public float getProgress() { private synchronized void setStateOnDAGCompletion() { DAGAppMasterState oldState = state; - if(isSession) { + if (isSession) { return; } - switch(currentDAG.getState()) { - case SUCCEEDED: - state = DAGAppMasterState.SUCCEEDED; - break; - case FAILED: - state = DAGAppMasterState.FAILED; - break; - case KILLED: - state = DAGAppMasterState.KILLED; - break; - case ERROR: - state = DAGAppMasterState.ERROR; - break; - default: - state = DAGAppMasterState.ERROR; - break; + switch (currentDAG.getState()) { + case SUCCEEDED: + state = DAGAppMasterState.SUCCEEDED; + break; + case FAILED: + state = DAGAppMasterState.FAILED; + break; + case KILLED: + state = DAGAppMasterState.KILLED; + break; + case ERROR: + state = DAGAppMasterState.ERROR; + break; + default: + state = DAGAppMasterState.ERROR; + break; } LOG.info("On DAG completion. Old state: " + oldState + " new state: " + state); @@ -1285,7 +1280,7 @@ public Void run() throws Exception { } public String submitDAGToAppMaster(DAGPlan dagPlan, - Map additionalResources) throws TezException { + Map additionalResources) throws TezException { if (sessionStopped.get()) { throw new SessionNotRunning("AM unable to accept new DAG submissions." + " In the process of shutting down"); @@ -1308,9 +1303,9 @@ public String submitDAGToAppMaster(DAGPlan dagPlan, + " incompatible with the client. " + versionMismatchDiagnostics); } if (state.equals(DAGAppMasterState.ERROR) - || sessionStopped.get()) { + || sessionStopped.get()) { throw new SessionNotRunning("AM unable to accept new DAG submissions." - + " In the process of shutting down"); + + " In the process of shutting down"); } if (currentDAG != null && !currentDAG.isComplete()) { @@ -1365,7 +1360,7 @@ private Map getAdditionalLocalResourceDiff( } private boolean isSameFile(DAG dag, final String fileName, - final LocalResource oldLr, final LocalResource newLr) throws TezException { + final LocalResource oldLr, final LocalResource newLr) throws TezException { try { return oldLr.equals(newLr) || dag.getDagUGI().doAs(new PrivilegedExceptionAction() { @Override @@ -1648,7 +1643,6 @@ public String getContainerLauncherClassName(int launcherId) { return containerLauncherManager.getContainerLauncherClassName(launcherId); } - @Override public HadoopShim getHadoopShim() { return hadoopShim; @@ -1667,7 +1661,7 @@ public Map getApplicationACLs() { public TezDAGID getCurrentDAGID() { try { rLock.lock(); - if(dag != null) { + if (dag != null) { return dag.getID(); } return null; @@ -1723,6 +1717,7 @@ private static class ServiceWithDependency implements ServiceStateChangeListener ServiceWithDependency(Service service) { this.service = service; } + Service service; List dependencies = new ArrayList(); AtomicInteger dependenciesStarted = new AtomicInteger(0); @@ -1731,15 +1726,15 @@ private static class ServiceWithDependency implements ServiceStateChangeListener @Override public void stateChanged(Service dependency) { - if(LOG.isDebugEnabled()) { + if (LOG.isDebugEnabled()) { LOG.debug("Service dependency: " + dependency.getName() + " notify" + - " for service: " + service.getName()); + " for service: " + service.getName()); } Throwable dependencyError = dependency.getFailureCause(); if (dependencyError != null) { - synchronized(this) { + synchronized (this) { dependenciesFailed = true; - if(LOG.isDebugEnabled()) { + if (LOG.isDebugEnabled()) { LOG.debug("Service: " + service.getName() + " will fail to start" + " as dependent service " + dependency.getName() + " failed to start: " + dependencyError); @@ -1747,9 +1742,9 @@ public void stateChanged(Service dependency) { this.notifyAll(); } } else if (dependency.isInState(Service.STATE.STARTED)) { - if(dependenciesStarted.incrementAndGet() == dependencies.size()) { - synchronized(this) { - if(LOG.isDebugEnabled()) { + if (dependenciesStarted.incrementAndGet() == dependencies.size()) { + synchronized (this) { + if (LOG.isDebugEnabled()) { LOG.debug("Service: " + service.getName() + " notified to start"); } canStart = true; @@ -1760,10 +1755,10 @@ public void stateChanged(Service dependency) { } void start() throws InterruptedException { - if(dependencies.size() > 0) { - synchronized(this) { - while(!canStart) { - this.wait(1000*60*3L); + if (dependencies.size() > 0) { + synchronized (this) { + while (!canStart) { + this.wait(1000 * 60 * 3L); if (dependenciesFailed) { throw new TezUncheckedException("Skipping service start for " + service.getName() @@ -1772,14 +1767,14 @@ void start() throws InterruptedException { } } } - if(LOG.isDebugEnabled()) { + if (LOG.isDebugEnabled()) { LOG.debug("Service: " + service.getName() + " trying to start"); } - for(Service dependency : dependencies) { - if(!dependency.isInState(Service.STATE.STARTED)){ + for (Service dependency : dependencies) { + if (!dependency.isInState(Service.STATE.STARTED)) { LOG.info("Service: " + service.getName() + " not started because " - + " service: " + dependency.getName() + - " is in state: " + dependency.getServiceState()); + + " service: " + dependency.getName() + + " is in state: " + dependency.getServiceState()); return; } } @@ -1791,15 +1786,16 @@ private static class ServiceThread extends Thread { final ServiceWithDependency serviceWithDependency; final Map services; volatile Throwable error = null; + public ServiceThread(ServiceWithDependency serviceWithDependency, - Map services) { + Map services) { this.serviceWithDependency = serviceWithDependency; this.services = services; this.setName("ServiceThread:" + serviceWithDependency.service.getName()); } public void run() { - if(LOG.isDebugEnabled()) { + if (LOG.isDebugEnabled()) { LOG.debug("Starting thread " + serviceWithDependency.service.getName()); } long start = System.currentTimeMillis(); @@ -1815,12 +1811,12 @@ public void run() { error = t; notifyDependentServices(); } finally { - if(LOG.isDebugEnabled()) { + if (LOG.isDebugEnabled()) { LOG.debug("Service: " + serviceWithDependency.service.getName() + " started in " + (System.currentTimeMillis() - start) + "ms"); } } - if(LOG.isDebugEnabled()) { + if (LOG.isDebugEnabled()) { LOG.debug("Service thread completed for " + serviceWithDependency.service.getName()); } @@ -1841,27 +1837,27 @@ void startServices() { List threads = new ArrayList(); LOG.debug("Begin parallel start"); - for(ServiceWithDependency sd : services.values()) { + for (ServiceWithDependency sd : services.values()) { // start the service. If this fails that service // will be stopped and an exception raised ServiceThread st = new ServiceThread(sd, services); threads.add(st); } - for(ServiceThread st : threads) { + for (ServiceThread st : threads) { st.start(); } - for(ServiceThread st : threads) { - if(LOG.isDebugEnabled()) { + for (ServiceThread st : threads) { + if (LOG.isDebugEnabled()) { LOG.debug("Waiting for service thread to join for " + st.getName()); } st.join(); - if(st.error != null && firstError == null) { + if (st.error != null && firstError == null) { firstError = st.error; } } - if(firstError != null) { + if (firstError != null) { throw ServiceStateException.convert(firstError); } LOG.debug("End parallel start"); @@ -2005,7 +2001,7 @@ public void serviceStart() throws Exception { + ", isCompleted=" + recoveredDAGData.isCompleted + ", isNonRecoverable=" + recoveredDAGData.nonRecoverable + ", state=" + (recoveredDAGData.dagState == null ? "null" : - recoveredDAGData.dagState) + recoveredDAGData.dagState) + ", failureReason=" + recoveredDAGData.reason); _updateLoggers(recoveredDAGData.recoveredDAG, ""); if (recoveredDAGData.nonRecoverable) { @@ -2060,7 +2056,7 @@ public void serviceStart() throws Exception { if (!isSession) { // No dag recovered - in non-session, just restart the original DAG dagCounter.set(0); - assert(dagPlan != null); + assert (dagPlan != null); startDAG(dagPlan, null); } } @@ -2103,10 +2099,8 @@ public void run() { } }, clientAMHeartbeatTimeoutIntervalMillis, TimeUnit.MILLISECONDS); } - } - private void initiateStop() { taskSchedulerManager.initiateStop(); } @@ -2132,7 +2126,7 @@ public void serviceStop() throws Exception { TezConfiguration.TEZ_AM_STAGING_SCRATCH_DATA_AUTO_DELETE, TezConfiguration.TEZ_AM_STAGING_SCRATCH_DATA_AUTO_DELETE_DEFAULT); LOG.debug("Checking whether tez scratch data dir should be deleted, deleteTezScratchData={}", - deleteTezScratchData); + deleteTezScratchData); if (deleteTezScratchData && this.taskSchedulerManager != null && this.taskSchedulerManager.hasUnregistered()) { // Delete tez scratch data dir @@ -2176,7 +2170,7 @@ public void handle(DAGEvent event) { if (dag == null || eventDagIndex != dag.getID().getId()) { return; // event not relevant any more } - ((EventHandler)dag).handle(event); + ((EventHandler) dag).handle(event); } } @@ -2193,7 +2187,7 @@ public void handle(TaskEvent event) { Task task = dag.getVertex(event.getVertexID()). getTask(event.getTaskID()); - ((EventHandler)task).handle(event); + ((EventHandler) task).handle(event); } } @@ -2210,7 +2204,7 @@ public void handle(SpeculatorEvent event) { } private class TaskAttemptEventDispatcher - implements EventHandler { + implements EventHandler { @SuppressWarnings("unchecked") @Override public void handle(TaskAttemptEvent event) { @@ -2229,7 +2223,7 @@ public void handle(TaskAttemptEvent event) { } private class VertexEventDispatcher - implements EventHandler { + implements EventHandler { @SuppressWarnings("unchecked") @Override public void handle(VertexEvent event) { @@ -2260,7 +2254,7 @@ private long checkAndHandleDAGClientTimeout() throws TezException { if (currentTime < nextExpiry) { // reschedule timer to 1 sec after the next expiry window // to ensure that we time out as intended if there are no heartbeats - return ((nextExpiry+1000) - currentTime); + return ((nextExpiry + 1000) - currentTime); } String message = "Client-to-AM Heartbeat timeout interval expired, shutting down AM as client" @@ -2371,7 +2365,7 @@ public static void main(String[] args) { TezCommonUtils.getTrimmedStrings(System.getenv(Environment.LOG_DIRS.name())), clientVersion, credentials, jobUserName, amPluginDescriptorProto); ShutdownHookManager.get().addShutdownHook( - new DAGAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY); + new DAGAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY); // log the system properties if (LOG.isInfoEnabled()) { @@ -2382,7 +2376,6 @@ public static void main(String[] args) { } initAndStartAppMaster(appMaster, conf); - } catch (Throwable t) { LOG.error("Error starting DAGAppMaster", t); System.exit(1); @@ -2393,12 +2386,14 @@ public static void main(String[] args) { // close of the JVM. static class DAGAppMasterShutdownHook implements Runnable { DAGAppMaster appMaster; + DAGAppMasterShutdownHook(DAGAppMaster appMaster) { this.appMaster = appMaster; } + public void run() { LOG.info("DAGAppMasterShutdownHook invoked"); - if(appMaster.getServiceState() == STATE.STOPPED) { + if (appMaster.getServiceState() == STATE.STOPPED) { LOG.debug("DAGAppMaster already stopped. Ignoring signal"); synchronized (appMaster.shutdownHandlerRunning) { try { @@ -2414,7 +2409,7 @@ public void run() { return; } - if(appMaster.getServiceState() == STATE.STARTED) { + if (appMaster.getServiceState() == STATE.STARTED) { // Notify TaskScheduler that a SIGTERM has been received so that it // unregisters quickly with proper status LOG.info("DAGAppMaster received a signal. Signaling TaskScheduler"); @@ -2423,14 +2418,13 @@ public void run() { if (EnumSet.of(DAGAppMasterState.NEW, DAGAppMasterState.INITED, DAGAppMasterState.IDLE).contains(appMaster.state)) { - // DAG not in a final state. Must have receive a KILL signal + // DAG not in a final state. Must have receive a KILL signal appMaster.state = DAGAppMasterState.KILLED; } else if (appMaster.state == DAGAppMasterState.RUNNING) { appMaster.state = DAGAppMasterState.ERROR; } appMaster.stop(); - } } @@ -2492,7 +2486,7 @@ private void startDAG(DAGPlan dagPlan, Map additionalAMRe String timeStamp = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(Calendar.getInstance().getTime()); System.err.println(timeStamp + " Running Dag: " + newDAG.getID()); - System.out.println(timeStamp + " Running Dag: "+ newDAG.getID()); + System.out.println(timeStamp + " Running Dag: " + newDAG.getID()); // Job name is the same as the app name until we support multiple dags // for an app later @@ -2504,14 +2498,14 @@ private void startDAG(DAGPlan dagPlan, Map additionalAMRe TezConfiguration.TEZ_DAG_HISTORY_LOGGING_ENABLED_DEFAULT); submittedEvent.setHistoryLoggingEnabled(dagLoggingEnabled); try { - appMasterUgi.doAs(new PrivilegedExceptionAction() { - @Override - public Void run() throws Exception { - historyEventHandler.handleCriticalEvent( - new DAGHistoryEvent(newDAG.getID(), submittedEvent)); - return null; - } - }); + appMasterUgi.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + historyEventHandler.handleCriticalEvent( + new DAGHistoryEvent(newDAG.getID(), submittedEvent)); + return null; + } + }); } catch (IOException e) { throw new TezUncheckedException(e); } catch (InterruptedException e) { @@ -2530,7 +2524,7 @@ private void startVertexServices(DAG dag) { } void stopVertexServices(DAG dag) { - for (Vertex v: dag.getVertices().values()) { + for (Vertex v : dag.getVertices().values()) { v.stopServices(); } } @@ -2576,7 +2570,7 @@ public List run() throws Exception { } public static void initAndStartAppMaster(final DAGAppMaster appMaster, - final Configuration conf) throws IOException, + final Configuration conf) throws IOException, InterruptedException { // Do not automatically close FileSystem objects so that in case of @@ -2664,11 +2658,10 @@ static void parseAllPlugins( tezYarnEnabled, uberEnabled, defaultPayload); } - @VisibleForTesting public static void parsePlugin(List resultList, - BiMap pluginMap, List namedEntityDescriptorProtos, - boolean tezYarnEnabled, boolean uberEnabled, UserPayload defaultPayload) { + BiMap pluginMap, List namedEntityDescriptorProtos, + boolean tezYarnEnabled, boolean uberEnabled, UserPayload defaultPayload) { if (tezYarnEnabled) { // Default classnames will be populated by individual components diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMasterState.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMasterState.java index 238ff4ca24..ec2d10f7af 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMasterState.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMasterState.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/HeartbeatHandlerBase.java b/tez-dag/src/main/java/org/apache/tez/dag/app/HeartbeatHandlerBase.java index b5ded81500..d8c7ad4de3 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/HeartbeatHandlerBase.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/HeartbeatHandlerBase.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app; @@ -30,17 +30,16 @@ public abstract class HeartbeatHandlerBase extends AbstractService { - protected int timeOut; protected int timeOutCheckInterval; protected Thread timeOutCheckerThread; private final String name; - + @SuppressWarnings("rawtypes") protected final EventHandler eventHandler; protected final Clock clock; protected final AppContext appContext; - + private ConcurrentMap runningMap; public HeartbeatHandlerBase(AppContext appContext, int expectedConcurrency, String name) { @@ -73,67 +72,67 @@ public void serviceStop() { timeOutCheckerThread.interrupt(); } } - + protected Runnable createPingChecker() { return new PingChecker(); } + protected abstract int getConfiguredTimeout(Configuration conf); + protected abstract int getConfiguredTimeoutCheckInterval(Configuration conf); - + public void progressing(T id) { ReportTime time = runningMap.get(id); if (time != null) { time.setLastProgress(clock.getTime()); } } - + public void pinged(T id) { ReportTime time = runningMap.get(id); if (time != null) { time.setLastPing(clock.getTime()); } } - + public void register(T id) { runningMap.put(id, new ReportTime(clock.getTime())); } - + public void unregister(T id) { runningMap.remove(id); } - - - + protected static class ReportTime { private long lastPing; private long lastProgress; - + public ReportTime(long time) { setLastProgress(time); } - + public synchronized void setLastPing(long time) { lastPing = time; } - + public synchronized void setLastProgress(long time) { lastProgress = time; lastPing = time; } - + public synchronized long getLastPing() { return lastPing; } - + public synchronized long getLastProgress() { return lastProgress; } } - + protected abstract boolean hasTimedOut(ReportTime report, long currentTime); - + protected abstract void handleTimeOut(T t); - + private class PingChecker implements Runnable { @Override @@ -146,8 +145,8 @@ public void run() { long currentTime = clock.getTime(); while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - if(hasTimedOut(entry.getValue(), currentTime)) { + Map.Entry entry = iterator.next(); + if (hasTimedOut(entry.getValue(), currentTime)) { // Timed out. Removed from list and send out an event. iterator.remove(); handleTimeOut(entry.getKey()); @@ -161,5 +160,4 @@ public void run() { } } } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/LocalDAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/LocalDAGAppMaster.java index e0c8443577..8f0df43b7e 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/LocalDAGAppMaster.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/LocalDAGAppMaster.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -32,9 +32,9 @@ public class LocalDAGAppMaster extends DAGAppMaster { public LocalDAGAppMaster(ApplicationAttemptId applicationAttemptId, ContainerId containerId, - String nmHost, int nmPort, int nmHttpPort, Clock clock, long appSubmitTime, boolean isSession, - String workingDirectory, String[] localDirs, String[] logDirs, String clientVersion, - Credentials credentials, String jobUserName, AMPluginDescriptorProto pluginDescriptorProto) { + String nmHost, int nmPort, int nmHttpPort, Clock clock, long appSubmitTime, boolean isSession, + String workingDirectory, String[] localDirs, String[] logDirs, String clientVersion, + Credentials credentials, String jobUserName, AMPluginDescriptorProto pluginDescriptorProto) { super(applicationAttemptId, containerId, nmHost, nmPort, nmHttpPort, clock, appSubmitTime, isSession, workingDirectory, localDirs, logDirs, clientVersion, credentials, jobUserName, pluginDescriptorProto); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java b/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java index ce3b62bbd5..1c0d1f9d97 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/RecoveryParser.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -78,10 +78,9 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.tez.common.Preconditions; - /** * RecoverParser is mainly for Tez AM Recovery. It would read the recovery events. (summary & non-summary) - * + * */ public class RecoveryParser { @@ -95,9 +94,9 @@ public class RecoveryParser { private final int currentAttemptId; public RecoveryParser(DAGAppMaster dagAppMaster, - FileSystem recoveryFS, - Path recoveryDataDir, - int currentAttemptId) throws IOException { + FileSystem recoveryFS, + Path recoveryDataDir, + int currentAttemptId) throws IOException { this.dagAppMaster = dagAppMaster; this.recoveryFS = recoveryFS; this.recoveryDataDir = recoveryDataDir; @@ -161,7 +160,7 @@ public void checkRecoverableNonSummary() { // vertex has finished committing TezVertexID vertexId = entry.getKey(); boolean commitFinished = entry.getValue(); - if(commitFinished + if (commitFinished && (!vertexRecoveryDataMap.containsKey(vertexId) || vertexRecoveryDataMap.get(vertexId).getVertexFinishedEvent() == null)) { this.nonRecoverable = true; @@ -174,7 +173,7 @@ public void checkRecoverableNonSummary() { // vertex has finished committing TezVertexID vertexId = entry.getKey(); boolean commitFinished = entry.getValue(); - if(commitFinished + if (commitFinished && (!vertexRecoveryDataMap.containsKey(vertexId) || vertexRecoveryDataMap.get(vertexId).getVertexFinishedEvent() == null)) { this.nonRecoverable = true; @@ -339,14 +338,13 @@ private static HistoryEvent getNextEvent(CodedInputStream inputStream) default: throw new IOException("Invalid data found, unknown event type " + eventType); - } try { event.fromProtoStream(inputStream); } catch (EOFException eof) { return null; } - LOG.debug("Parsed event from input stream, eventType={}, event={}", + LOG.debug("Parsed event from input stream, eventType={}, event={}", eventType, event); return event; } @@ -369,14 +367,14 @@ public static List parseDAGRecoveryFile(FSDataInputStream inputStr } public static List readRecoveryEvents(TezConfiguration tezConf, ApplicationId appId, - int attempt) throws IOException { + int attempt) throws IOException { Path tezSystemStagingDir = TezCommonUtils.getTezSystemStagingPath(tezConf, appId.toString()); Path recoveryDataDir = TezCommonUtils.getRecoveryPath(tezSystemStagingDir, tezConf); FileSystem fs = tezSystemStagingDir.getFileSystem(tezConf); List historyEvents = new ArrayList(); - for (int i=1; i <= attempt; ++i) { + for (int i = 1; i <= attempt; ++i) { Path currentAttemptRecoveryDataDir = TezCommonUtils.getAttemptRecoveryPath(recoveryDataDir, i); Path recoveryFilePath = @@ -438,7 +436,7 @@ private FSDataInputStream getSummaryStream(Path summaryPath) } private Path getDAGRecoveryFilePath(Path recoveryDataDir, - TezDAGID dagID) { + TezDAGID dagID) { return new Path(recoveryDataDir, dagID.toString() + TezConstants.DAG_RECOVERY_RECOVER_FILE_SUFFIX); } @@ -710,7 +708,7 @@ public DAGRecoveryData parseRecoveryData() throws IOException { // Set counter for next set of DAGs & update dagNames Set in DAGAppMaster dagAppMaster.setDAGCounter(dagCounter); - for (DAGSummaryData dagSummaryData: dagSummaryDataMap.values()){ + for (DAGSummaryData dagSummaryData : dagSummaryDataMap.values()) { dagAppMaster.dagIDs.add(dagSummaryData.dagId.toString()); } @@ -780,7 +778,7 @@ public DAGRecoveryData parseRecoveryData() throws IOException { recoveredDAGData.recoveredDAG = dagAppMaster.createDAG(submittedEvent.getDAGPlan(), lastInProgressDAG); recoveredDAGData.cumulativeAdditionalResources = submittedEvent - .getCumulativeAdditionalLocalResources(); + .getCumulativeAdditionalLocalResources(); recoveredDAGData.recoveredDagID = recoveredDAGData.recoveredDAG.getID(); dagAppMaster.setCurrentDAG(recoveredDAGData.recoveredDAG); if (recoveredDAGData.nonRecoverable) { @@ -788,63 +786,54 @@ public DAGRecoveryData parseRecoveryData() throws IOException { } break; case DAG_INITIALIZED: - recoveredDAGData.dagInitedEvent = (DAGInitializedEvent)event; + recoveredDAGData.dagInitedEvent = (DAGInitializedEvent) event; break; case DAG_STARTED: - recoveredDAGData.dagStartedEvent= (DAGStartedEvent)event; + recoveredDAGData.dagStartedEvent = (DAGStartedEvent) event; break; case DAG_FINISHED: - recoveredDAGData.dagFinishedEvent = (DAGFinishedEvent)event; + recoveredDAGData.dagFinishedEvent = (DAGFinishedEvent) event; skipAllOtherEvents = true; - break; + break; case DAG_COMMIT_STARTED: case VERTEX_GROUP_COMMIT_STARTED: - case VERTEX_GROUP_COMMIT_FINISHED: - case CONTAINER_LAUNCHED: - { + case VERTEX_GROUP_COMMIT_FINISHED: + case CONTAINER_LAUNCHED: { // Nothing to do for now break; } - case DAG_KILL_REQUEST: - { + case DAG_KILL_REQUEST: { break; } - case VERTEX_INITIALIZED: - - { - VertexInitializedEvent vertexInitEvent = (VertexInitializedEvent)event; + case VERTEX_INITIALIZED: { + VertexInitializedEvent vertexInitEvent = (VertexInitializedEvent) event; VertexRecoveryData vertexRecoveryData = recoveredDAGData.maybeCreateVertexRecoveryData(vertexInitEvent.getVertexID()); vertexRecoveryData.vertexInitedEvent = vertexInitEvent; break; } - case VERTEX_CONFIGURE_DONE: - { - VertexConfigurationDoneEvent reconfigureDoneEvent = (VertexConfigurationDoneEvent)event; + case VERTEX_CONFIGURE_DONE: { + VertexConfigurationDoneEvent reconfigureDoneEvent = (VertexConfigurationDoneEvent) event; VertexRecoveryData vertexRecoveryData = recoveredDAGData.maybeCreateVertexRecoveryData(reconfigureDoneEvent.getVertexID()); vertexRecoveryData.vertexConfigurationDoneEvent = reconfigureDoneEvent; break; } - case VERTEX_STARTED: - { - VertexStartedEvent vertexStartedEvent = (VertexStartedEvent)event; + case VERTEX_STARTED: { + VertexStartedEvent vertexStartedEvent = (VertexStartedEvent) event; VertexRecoveryData vertexRecoveryData = recoveredDAGData.vertexRecoveryDataMap.get(vertexStartedEvent.getVertexID()); Preconditions.checkArgument(vertexRecoveryData != null, "No VertexInitializedEvent before VertexStartedEvent"); vertexRecoveryData.vertexStartedEvent = vertexStartedEvent; break; } - case VERTEX_COMMIT_STARTED: - { + case VERTEX_COMMIT_STARTED: { break; } - case VERTEX_FINISHED: - { - VertexFinishedEvent vertexFinishedEvent = (VertexFinishedEvent)event; + case VERTEX_FINISHED: { + VertexFinishedEvent vertexFinishedEvent = (VertexFinishedEvent) event; VertexRecoveryData vertexRecoveryData = recoveredDAGData.maybeCreateVertexRecoveryData(vertexFinishedEvent.getVertexID()); vertexRecoveryData.vertexFinishedEvent = vertexFinishedEvent; break; } - case TASK_STARTED: - { + case TASK_STARTED: { TaskStartedEvent taskStartedEvent = (TaskStartedEvent) event; VertexRecoveryData vertexRecoveryData = recoveredDAGData.vertexRecoveryDataMap.get(taskStartedEvent.getVertexID()); Preconditions.checkArgument(vertexRecoveryData != null, @@ -853,8 +842,7 @@ public DAGRecoveryData parseRecoveryData() throws IOException { taskRecoveryData.taskStartedEvent = taskStartedEvent; break; } - case TASK_FINISHED: - { + case TASK_FINISHED: { TaskFinishedEvent taskFinishedEvent = (TaskFinishedEvent) event; VertexRecoveryData vertexRecoveryData = recoveredDAGData.vertexRecoveryDataMap.get(taskFinishedEvent.getVertexID()); Preconditions.checkArgument(vertexRecoveryData != null, @@ -863,9 +851,8 @@ public DAGRecoveryData parseRecoveryData() throws IOException { taskRecoveryData.taskFinishedEvent = taskFinishedEvent; break; } - case TASK_ATTEMPT_STARTED: - { - TaskAttemptStartedEvent taStartedEvent = (TaskAttemptStartedEvent)event; + case TASK_ATTEMPT_STARTED: { + TaskAttemptStartedEvent taStartedEvent = (TaskAttemptStartedEvent) event; VertexRecoveryData vertexRecoveryData = recoveredDAGData.vertexRecoveryDataMap.get( taStartedEvent.getVertexID()); Preconditions.checkArgument(vertexRecoveryData != null, @@ -878,9 +865,8 @@ public DAGRecoveryData parseRecoveryData() throws IOException { taRecoveryData.taStartedEvent = taStartedEvent; break; } - case TASK_ATTEMPT_FINISHED: - { - TaskAttemptFinishedEvent taFinishedEvent = (TaskAttemptFinishedEvent)event; + case TASK_ATTEMPT_FINISHED: { + TaskAttemptFinishedEvent taFinishedEvent = (TaskAttemptFinishedEvent) event; VertexRecoveryData vertexRecoveryData = recoveredDAGData.vertexRecoveryDataMap.get( taFinishedEvent.getVertexID()); Preconditions.checkArgument(vertexRecoveryData != null, @@ -922,10 +908,10 @@ public static class VertexRecoveryData { @VisibleForTesting public VertexRecoveryData(VertexInitializedEvent vertexInitedEvent, - VertexConfigurationDoneEvent vertexReconfigureDoneEvent, - VertexStartedEvent vertexStartedEvent, - VertexFinishedEvent vertexFinishedEvent, - Map taskRecoveryDataMap, boolean commited) { + VertexConfigurationDoneEvent vertexReconfigureDoneEvent, + VertexStartedEvent vertexStartedEvent, + VertexFinishedEvent vertexFinishedEvent, + Map taskRecoveryDataMap, boolean commited) { super(); this.vertexInitedEvent = vertexInitedEvent; this.vertexConfigurationDoneEvent = vertexReconfigureDoneEvent; @@ -938,7 +924,7 @@ public VertexRecoveryData(VertexInitializedEvent vertexInitedEvent, public VertexRecoveryData(boolean committed) { this.commited = committed; } - + public VertexInitializedEvent getVertexInitedEvent() { return vertexInitedEvent; } @@ -1025,8 +1011,8 @@ public TaskRecoveryData() { @VisibleForTesting public TaskRecoveryData(TaskStartedEvent taskStartedEvent, - TaskFinishedEvent taskFinishedEvent, - Map taRecoveryDataMap) { + TaskFinishedEvent taskFinishedEvent, + Map taRecoveryDataMap) { super(); this.taskStartedEvent = taskStartedEvent; this.taskFinishedEvent = taskFinishedEvent; @@ -1071,7 +1057,7 @@ public TaskAttemptRecoveryData() { @VisibleForTesting public TaskAttemptRecoveryData(TaskAttemptStartedEvent taStartedEvent, - TaskAttemptFinishedEvent taFinishedEvent) { + TaskAttemptFinishedEvent taFinishedEvent) { super(); this.taStartedEvent = taStartedEvent; this.taFinishedEvent = taFinishedEvent; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/ServicePluginLifecycleAbstractService.java b/tez-dag/src/main/java/org/apache/tez/dag/app/ServicePluginLifecycleAbstractService.java index dac1b82cca..610b1b4336 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/ServicePluginLifecycleAbstractService.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/ServicePluginLifecycleAbstractService.java @@ -20,6 +20,7 @@ /** * Provides service lifecycle management over ServicePlugins using {@link AbstractService} + * * @param */ public class ServicePluginLifecycleAbstractService extends AbstractService { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptEventInfo.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptEventInfo.java index d6b03e4ecd..572c73560f 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptEventInfo.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskAttemptEventInfo.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -25,17 +25,17 @@ public class TaskAttemptEventInfo { private final int nextFromEventId; private final int nextPreRoutedFromEventId; private final List events; - + public TaskAttemptEventInfo(int nextFromEventId, List events, int nextPreRoutedEventId) { this.nextFromEventId = nextFromEventId; this.nextPreRoutedFromEventId = nextPreRoutedEventId; this.events = events; } - + public int getNextFromEventId() { return nextFromEventId; } - + public int getNextPreRoutedFromEventId() { return nextPreRoutedFromEventId; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java index 4f9e8723aa..24ff462f1e 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorContextImpl.java @@ -216,7 +216,7 @@ public int getVertexRunningTaskCount(String vertexName) { @Override public long getFirstAttemptStartTime(String vertexName, int taskIndex) { Objects.requireNonNull(vertexName, "VertexName must be specified"); - Preconditions.checkArgument(taskIndex >=0, "TaskIndex must be > 0"); + Preconditions.checkArgument(taskIndex >= 0, "TaskIndex must be > 0"); DAG dag = getDag(); Vertex vertex = dag.getVertex(vertexName); Task task = vertex.getTask(taskIndex); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManager.java index 51895f4afd..df06774021 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManager.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManager.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -80,7 +80,6 @@ import org.apache.tez.dag.records.TezVertexID; import org.apache.tez.runtime.api.impl.TezEvent; - @SuppressWarnings("unchecked") @InterfaceAudience.Private public class TaskCommunicatorManager extends AbstractService implements @@ -92,7 +91,7 @@ public class TaskCommunicatorManager extends AbstractService implements private final AppContext context; private final TaskCommunicatorWrapper[] taskCommunicators; private final TaskCommunicatorContext[] taskCommunicatorContexts; - protected final ServicePluginLifecycleAbstractService []taskCommunicatorServiceWrappers; + protected final ServicePluginLifecycleAbstractService[] taskCommunicatorServiceWrappers; protected final TaskHeartbeatHandler taskHeartbeatHandler; protected final ContainerHeartbeatHandler containerHeartbeatHandler; @@ -107,6 +106,7 @@ public class TaskCommunicatorManager extends AbstractService implements // Defined primarily to work around ConcurrentMaps not accepting null values private static final class ContainerInfo { TezTaskAttemptID taskAttemptId; + ContainerInfo(TezTaskAttemptID taskAttemptId) { this.taskAttemptId = taskAttemptId; } @@ -114,7 +114,6 @@ private static final class ContainerInfo { private static final ContainerInfo NULL_CONTAINER_INFO = new ContainerInfo(null); - @VisibleForTesting @InterfaceAudience.Private /** @@ -146,7 +145,7 @@ public TaskCommunicatorManager(AppContext context, this.taskCommunicators = new TaskCommunicatorWrapper[taskCommunicatorDescriptors.size()]; this.taskCommunicatorContexts = new TaskCommunicatorContext[taskCommunicatorDescriptors.size()]; this.taskCommunicatorServiceWrappers = new ServicePluginLifecycleAbstractService[taskCommunicatorDescriptors.size()]; - for (int i = 0 ; i < taskCommunicatorDescriptors.size() ; i++) { + for (int i = 0; i < taskCommunicatorDescriptors.size(); i++) { UserPayload userPayload = taskCommunicatorDescriptors.get(i).getUserPayload(); taskCommunicatorContexts[i] = new TaskCommunicatorContextImpl(context, this, userPayload, i); taskCommunicators[i] = new TaskCommunicatorWrapper(createTaskCommunicator(taskCommunicatorDescriptors.get(i), i)); @@ -159,7 +158,7 @@ public TaskCommunicatorManager(AppContext context, @Override public void serviceStart() { // TODO Why is init tied to serviceStart - for (int i = 0 ; i < taskCommunicators.length ; i++) { + for (int i = 0; i < taskCommunicators.length; i++) { taskCommunicatorServiceWrappers[i].init(getConfig()); taskCommunicatorServiceWrappers[i].start(); } @@ -167,7 +166,7 @@ public void serviceStart() { @Override public void serviceStop() { - for (int i = 0 ; i < taskCommunicators.length ; i++) { + for (int i = 0; i < taskCommunicators.length; i++) { taskCommunicatorServiceWrappers[i].stop(); } } @@ -201,7 +200,7 @@ TaskCommunicator createUberTaskCommunicator(TaskCommunicatorContext taskCommunic @VisibleForTesting TaskCommunicator createCustomTaskCommunicator(TaskCommunicatorContext taskCommunicatorContext, NamedEntityDescriptor taskCommDescriptor) - throws TezException { + throws TezException { LOG.info("Creating TaskCommunicator {}:{} " + taskCommDescriptor.getEntityName(), taskCommDescriptor.getClassName()); Class taskCommClazz = @@ -211,7 +210,8 @@ TaskCommunicator createCustomTaskCommunicator(TaskCommunicatorContext taskCommun Constructor ctor = taskCommClazz.getConstructor(TaskCommunicatorContext.class); return ctor.newInstance(taskCommunicatorContext); - } catch (NoSuchMethodException | InvocationTargetException | InstantiationException | IllegalAccessException e) { + } catch (NoSuchMethodException | InvocationTargetException | InstantiationException | + IllegalAccessException e) { throw new TezUncheckedException(e); } } @@ -277,17 +277,17 @@ public TaskHeartbeatResponse heartbeat(TaskHeartbeatRequest request) taskAttemptEvent = new TaskAttemptEventStatusUpdate(taskAttemptID, (TaskStatusUpdateEvent) tezEvent.getEvent()); } else if (eventType == EventType.TASK_ATTEMPT_COMPLETED_EVENT - || eventType == EventType.TASK_ATTEMPT_FAILED_EVENT - || eventType == EventType.TASK_ATTEMPT_KILLED_EVENT) { + || eventType == EventType.TASK_ATTEMPT_FAILED_EVENT + || eventType == EventType.TASK_ATTEMPT_KILLED_EVENT) { taFinishedEvents.add(tezEvent); } else { if (eventType == EventType.INPUT_READ_ERROR_EVENT) { readErrorReported = true; } if (eventType == EventType.DATA_MOVEMENT_EVENT - || eventType == EventType.COMPOSITE_DATA_MOVEMENT_EVENT - || eventType == EventType.ROOT_INPUT_INITIALIZER_EVENT - || eventType == EventType.VERTEX_MANAGER_EVENT) { + || eventType == EventType.COMPOSITE_DATA_MOVEMENT_EVENT + || eventType == EventType.ROOT_INPUT_INITIALIZER_EVENT + || eventType == EventType.VERTEX_MANAGER_EVENT) { taGeneratedEvents.add(tezEvent); } eventsForVertex.add(tezEvent); @@ -306,47 +306,47 @@ public TaskHeartbeatResponse heartbeat(TaskHeartbeatRequest request) for (TezEvent e : taFinishedEvents) { EventMetaData sourceMeta = e.getSourceInfo(); switch (e.getEventType()) { - case TASK_ATTEMPT_FAILED_EVENT: - case TASK_ATTEMPT_KILLED_EVENT: - TaskAttemptTerminationCause errCause = null; - switch (sourceMeta.getEventGenerator()) { - case INPUT: - errCause = TaskAttemptTerminationCause.INPUT_READ_ERROR; - break; - case PROCESSOR: - errCause = TaskAttemptTerminationCause.APPLICATION_ERROR; + case TASK_ATTEMPT_FAILED_EVENT: + case TASK_ATTEMPT_KILLED_EVENT: + TaskAttemptTerminationCause errCause = null; + switch (sourceMeta.getEventGenerator()) { + case INPUT: + errCause = TaskAttemptTerminationCause.INPUT_READ_ERROR; + break; + case PROCESSOR: + errCause = TaskAttemptTerminationCause.APPLICATION_ERROR; + break; + case OUTPUT: + errCause = TaskAttemptTerminationCause.OUTPUT_WRITE_ERROR; + break; + case SYSTEM: + errCause = TaskAttemptTerminationCause.FRAMEWORK_ERROR; + break; + default: + throw new TezUncheckedException("Unknown EventProducerConsumerType: " + + sourceMeta.getEventGenerator()); + } + if (e.getEventType() == EventType.TASK_ATTEMPT_FAILED_EVENT) { + TaskAttemptFailedEvent taskFailedEvent = (TaskAttemptFailedEvent) e.getEvent(); + sendEvent( + new TaskAttemptEventAttemptFailed(sourceMeta.getTaskAttemptID(), + TaskAttemptEventType.TA_FAILED, taskFailedEvent.getTaskFailureType(), + "Error: " + taskFailedEvent.getDiagnostics(), + errCause)); + } else { // Killed + TaskAttemptKilledEvent taskKilledEvent = (TaskAttemptKilledEvent) e.getEvent(); + sendEvent( + new TaskAttemptEventAttemptKilled(sourceMeta.getTaskAttemptID(), + "Error: " + taskKilledEvent.getDiagnostics(), errCause)); + } break; - case OUTPUT: - errCause = TaskAttemptTerminationCause.OUTPUT_WRITE_ERROR; - break; - case SYSTEM: - errCause = TaskAttemptTerminationCause.FRAMEWORK_ERROR; + case TASK_ATTEMPT_COMPLETED_EVENT: + sendEvent( + new TaskAttemptEvent(sourceMeta.getTaskAttemptID(), TaskAttemptEventType.TA_DONE)); break; default: - throw new TezUncheckedException("Unknown EventProducerConsumerType: " + - sourceMeta.getEventGenerator()); - } - if (e.getEventType() == EventType.TASK_ATTEMPT_FAILED_EVENT) { - TaskAttemptFailedEvent taskFailedEvent = (TaskAttemptFailedEvent) e.getEvent(); - sendEvent( - new TaskAttemptEventAttemptFailed(sourceMeta.getTaskAttemptID(), - TaskAttemptEventType.TA_FAILED, taskFailedEvent.getTaskFailureType(), - "Error: " + taskFailedEvent.getDiagnostics(), - errCause)); - } else { // Killed - TaskAttemptKilledEvent taskKilledEvent = (TaskAttemptKilledEvent) e.getEvent(); - sendEvent( - new TaskAttemptEventAttemptKilled(sourceMeta.getTaskAttemptID(), - "Error: " + taskKilledEvent.getDiagnostics(), errCause)); - } - break; - case TASK_ATTEMPT_COMPLETED_EVENT: - sendEvent( - new TaskAttemptEvent(sourceMeta.getTaskAttemptID(), TaskAttemptEventType.TA_DONE)); - break; - default: - throw new TezUncheckedException("Unhandled tez event type: " - + e.getEventType()); + throw new TezUncheckedException("Unhandled tez event type: " + + e.getEventType()); } } if (!eventsForVertex.isEmpty()) { @@ -423,7 +423,6 @@ public void vertexStateUpdateNotificationReceived(VertexStateUpdate event, int t } } - /** * Child checking whether it can commit. *

@@ -459,7 +458,7 @@ public void dagComplete(DAG dag) { // TODO TEZ-2336. Send a signal to containers indicating DAG completion. // Inform all communicators of the dagCompletion. - for (int i = 0 ; i < taskCommunicators.length ; i++) { + for (int i = 0; i < taskCommunicators.length; i++) { try { ((TaskCommunicatorContextImpl) taskCommunicatorContexts[i]).dagCompleteStart(dag); taskCommunicators[i].dagComplete(dag.getID().getId()); @@ -474,7 +473,6 @@ public void dagComplete(DAG dag) { msg, e)); } } - } @Override diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManagerInterface.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManagerInterface.java index 150977a94e..d4224ad525 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManagerInterface.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorManagerInterface.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app; @@ -27,6 +27,7 @@ import org.apache.tez.dag.app.dag.DAG; import org.apache.tez.dag.app.rm.container.AMContainerTask; import org.apache.tez.dag.records.TezTaskAttemptID; + /** * This class listens for changes to the state of a Task. */ @@ -35,9 +36,9 @@ public interface TaskCommunicatorManagerInterface { void registerRunningContainer(ContainerId containerId, int taskCommId); void registerTaskAttempt(AMContainerTask amContainerTask, ContainerId containerId, int taskCommId); - + void unregisterRunningContainer(ContainerId containerId, int taskCommId, ContainerEndReason endReason, String diagnostics); - + void unregisterTaskAttempt(TezTaskAttemptID attemptID, int taskCommId, TaskAttemptEndReason endReason, String diagnostics); void dagComplete(DAG dag); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorWrapper.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorWrapper.java index 4afb427623..3206061361 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorWrapper.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskCommunicatorWrapper.java @@ -37,7 +37,6 @@ public TaskCommunicatorWrapper(TaskCommunicator real) { this.real = real; } - public void registerRunningContainer(ContainerId containerId, String hostname, int port) throws Exception { real.registerRunningContainer(containerId, hostname, port); @@ -46,7 +45,6 @@ public void registerRunningContainer(ContainerId containerId, String hostname, i public void registerContainerEnd(ContainerId containerId, ContainerEndReason endReason, @Nullable String diagnostics) throws Exception { real.registerContainerEnd(containerId, endReason, diagnostics); - } public void registerRunningTaskAttempt(ContainerId containerId, TaskSpec taskSpec, @@ -91,5 +89,4 @@ public String getCompletedLogsUrl(TezTaskAttemptID attemptID, NodeId containerNo throws Exception { return real.getCompletedLogsUrl(attemptID, containerNodeId); } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java index 296073a426..eeb757c94c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java @@ -1,24 +1,23 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app; - import org.apache.hadoop.conf.Configuration; import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptFailed; @@ -27,12 +26,11 @@ import org.apache.tez.dag.records.TezTaskAttemptID; import org.apache.tez.runtime.api.TaskFailureType; - /** * This class keeps track of tasks that have already been launched. It * determines if a task is alive and running or marks a task as dead if it does * not hear from it for a long time. - * + * */ @SuppressWarnings({"unchecked"}) public class TaskHeartbeatHandler extends HeartbeatHandlerBase { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TezDagVersionInfo.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TezDagVersionInfo.java index 38b42a210f..93f42b5fe5 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/TezDagVersionInfo.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TezDagVersionInfo.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java index 48aee3103b..57e03f1f29 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TezTaskCommunicatorImpl.java @@ -78,7 +78,6 @@ public class TezTaskCommunicatorImpl extends TaskCommunicator { protected final ConcurrentMap attemptToContainerMap = new ConcurrentHashMap<>(); - protected final String tokenIdentifier; protected final Token sessionToken; protected final Configuration conf; @@ -115,8 +114,6 @@ void reset() { } } - - /** * Construct the service. */ @@ -207,7 +204,7 @@ public void registerRunningContainer(ContainerId containerId, String host, int p public void registerContainerEnd(ContainerId containerId, ContainerEndReason endReason, String diagnostics) { ContainerInfo containerInfo = registeredContainers.remove(containerId); if (containerInfo != null) { - synchronized(containerInfo) { + synchronized (containerInfo) { if (containerInfo.taskSpec != null && containerInfo.taskSpec.getTaskAttemptID() != null) { attemptToContainerMap.remove(containerInfo.taskSpec.getTaskAttemptID()); } @@ -248,11 +245,10 @@ public void registerRunningTaskAttempt(ContainerId containerId, TaskSpec taskSpe } } - @Override public void unregisterRunningTaskAttempt(TezTaskAttemptID taskAttemptID, TaskAttemptEndReason endReason, String diagnostics) { ContainerId containerId = attemptToContainerMap.remove(taskAttemptID); - if(containerId == null) { + if (containerId == null) { LOG.warn("Unregister task attempt: " + taskAttemptID + " from unknown container"); return; } @@ -352,8 +348,6 @@ public TezHeartbeatResponse heartbeat(TezHeartbeatRequest request) throws IOExce } } - - TezHeartbeatResponse response = new TezHeartbeatResponse(); TezTaskAttemptID taskAttemptID = request.getCurrentTaskAttemptID(); if (taskAttemptID != null) { @@ -387,7 +381,6 @@ public TezHeartbeatResponse heartbeat(TezHeartbeatRequest request) throws IOExce return response; } - // TODO Remove this method once we move to the Protobuf RPC engine @Override public long getProtocolVersion(String protocol, long clientVersion) throws IOException { @@ -472,4 +465,4 @@ protected ContainerId getContainerForAttempt(TezTaskAttemptID taskAttemptId) { public long getTotalUsedMemory() { return registeredContainers.values().stream().mapToLong(c -> c.usedMemory).sum(); } -} \ No newline at end of file +} diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java index 280966ef78..181212218c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag; @@ -44,9 +44,13 @@ public interface DAG extends DagInfo { TezDAGID getID(); + Map getLocalResources(); + String getName(); + DAGState getState(); + DAGReport getReport(); /** @@ -56,6 +60,7 @@ public interface DAG extends DagInfo { * @return job-counters and aggregate task-counters */ TezCounters getAllCounters(); + TezCounters getCachedCounters(); @SuppressWarnings("rawtypes") @@ -65,19 +70,30 @@ public interface DAG extends DagInfo { * Get Vertex by vertex name */ Vertex getVertex(String vertexName); - Map getVertices(); + + Map getVertices(); + Vertex getVertex(TezVertexID vertexId); + List getDiagnostics(); + int getSuccessfulVertices(); + float getProgress(); + float getCompletedTaskProgress(); + boolean isUber(); + String getUserName(); DAGPlan getJobPlan(); + DAGStatusBuilder getDAGStatus(Set statusOptions); + DAGStatusBuilder getDAGStatus(Set statusOptions, long timeout) throws TezException; + VertexStatusBuilder getVertexStatus(String vertexName, Set statusOptions); @@ -100,6 +116,6 @@ VertexStatusBuilder getVertexStatus(String vertexName, * @return the DAGScheduler that will schedule * this DAG, null if it doesn't exist */ - @Nullable DAGScheduler getDAGScheduler(); - + @Nullable + DAGScheduler getDAGScheduler(); } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGReport.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGReport.java index 49ab24ac6d..76b76b744c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGReport.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGReport.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag; import java.util.List; @@ -24,37 +24,66 @@ public abstract class DAGReport { public abstract ApplicationId getDAGId(); + public abstract DAGState getDAGState(); + public abstract float getMapProgress(); + public abstract float getReduceProgress(); + public abstract float getCleanupProgress(); + public abstract float getSetupProgress(); + public abstract long getSubmitTime(); + public abstract long getStartTime(); + public abstract long getFinishTime(); + public abstract String getUser(); + public abstract String getJobName(); + public abstract String getTrackingUrl(); + public abstract String getDiagnostics(); + public abstract String getJobFile(); + public abstract List getAMInfos(); + public abstract boolean isUber(); public abstract void setDAGId(ApplicationId dagId); + public abstract void setJobState(DAGState dagState); + public abstract void setMapProgress(float progress); + public abstract void setReduceProgress(float progress); + public abstract void setCleanupProgress(float progress); + public abstract void setSetupProgress(float progress); + public abstract void setSubmitTime(long submitTime); + public abstract void setStartTime(long startTime); + public abstract void setFinishTime(long finishTime); + public abstract void setUser(String user); + public abstract void setJobName(String jobName); + public abstract void setTrackingUrl(String trackingUrl); + public abstract void setDiagnostics(String diagnostics); + public abstract void setJobFile(String jobFile); + public abstract void setAMInfos(List amInfos); - public abstract void setIsUber(boolean isUber); + public abstract void setIsUber(boolean isUber); } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGScheduler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGScheduler.java index ada7867a6a..591abcf06c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGScheduler.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGScheduler.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag; @@ -37,9 +37,9 @@ private static class VertexInfo { this.concurrencyLimit = limit; } } - + Map vertexInfo = null; - + public void addVertexConcurrencyLimit(TezVertexID vId, int concurrency) { if (vertexInfo == null) { vertexInfo = new HashMap<>(); @@ -48,7 +48,7 @@ public void addVertexConcurrencyLimit(TezVertexID vId, int concurrency) { vertexInfo.put(vId, new VertexInfo(concurrency)); } } - + public void scheduleTask(DAGEventSchedulerUpdate event) { VertexInfo vInfo = null; if (vertexInfo != null) { @@ -56,7 +56,7 @@ public void scheduleTask(DAGEventSchedulerUpdate event) { } scheduleTaskWithLimit(event, vInfo); } - + private void scheduleTaskWithLimit(DAGEventSchedulerUpdate event, VertexInfo vInfo) { if (vInfo != null) { if (vInfo.concurrency >= vInfo.concurrencyLimit) { @@ -67,15 +67,15 @@ private void scheduleTaskWithLimit(DAGEventSchedulerUpdate event, VertexInfo vIn } scheduleTaskEx(event); } - + public void taskCompleted(DAGEventSchedulerUpdate event) { taskCompletedEx(event); if (vertexInfo != null) { VertexInfo vInfo = vertexInfo.get(event.getVertexID()); if (vInfo != null) { - if(vInfo.pendingAttempts.remove(event.getTaskAttemptID()) == null) { + if (vInfo.pendingAttempts.remove(event.getTaskAttemptID()) == null) { vInfo.concurrency--; - if(!vInfo.pendingAttempts.isEmpty()) { + if (!vInfo.pendingAttempts.isEmpty()) { Iterator i = vInfo.pendingAttempts.values().iterator(); DAGEventSchedulerUpdate nextTaskAttempt = i.next(); i.remove(); @@ -85,9 +85,9 @@ public void taskCompleted(DAGEventSchedulerUpdate event) { } } } - + public abstract void scheduleTaskEx(DAGEventSchedulerUpdate event); - + public abstract void taskCompletedEx(DAGEventSchedulerUpdate event); /** @@ -108,7 +108,6 @@ public int getPriorityLowLimit(final DAG dag, final Vertex vertex) { * @return the priority */ public int getPriorityHighLimit(final DAG dag, final Vertex vertex) { - return getPriorityLowLimit(dag, vertex) - 2; + return getPriorityLowLimit(dag, vertex) - 2; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGState.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGState.java index 1acc08db1d..0425c3d004 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGState.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGState.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag; public enum DAGState { @@ -24,7 +24,7 @@ public enum DAGState { SUCCEEDED, FAILED, KILLED, - ERROR, + ERROR, TERMINATING, COMMITTING, } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGTerminationCause.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGTerminationCause.java index b73cbe647b..53faa9d201 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGTerminationCause.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAGTerminationCause.java @@ -1,24 +1,23 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag; - /** * Represents proximate cause of a DAG transition to FAILED or KILLED. */ @@ -29,16 +28,16 @@ public enum DAGTerminationCause { /** A service plugin indicated an error */ SERVICE_PLUGIN_ERROR(DAGState.FAILED), - - /** A vertex failed. */ + + /** A vertex failed. */ VERTEX_FAILURE(DAGState.FAILED), - + /** DAG failed due as it had zero vertices. */ ZERO_VERTICES(DAGState.FAILED), - + /** DAG failed during init. */ INIT_FAILURE(DAGState.FAILED), - + /** DAG failed during output commit. */ COMMIT_FAILURE(DAGState.FAILED), diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/RootInputInitializerManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/RootInputInitializerManager.java index cfbdb19e30..f9404055bc 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/RootInputInitializerManager.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/RootInputInitializerManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -111,7 +111,7 @@ public RootInputInitializerManager(Vertex vertex, AppContext appContext, } public void runInputInitializers(List> - inputs) throws TezException { + inputs) throws TezException { for (RootInputLeafOutput input : inputs) { InputInitializerContext context = @@ -146,7 +146,7 @@ public void runInputInitializers(List - input, final InputInitializerContext context) throws TezException { + input, final InputInitializerContext context) throws TezException { try { return dagUgi.doAs(new PrivilegedExceptionAction() { @Override @@ -175,7 +175,7 @@ public void handleInitializerEvents(List events) { for (TezEvent tezEvent : events) { Preconditions.checkState(tezEvent.getEvent() instanceof InputInitializerEvent); - InputInitializerEvent event = (InputInitializerEvent)tezEvent.getEvent(); + InputInitializerEvent event = (InputInitializerEvent) tezEvent.getEvent(); Preconditions.checkState(vertex.getName().equals(event.getTargetVertexName()), "Received event for incorrect vertex"); Objects.requireNonNull(event.getTargetInputName(), "target input name must be set"); @@ -203,7 +203,6 @@ public void handleInitializerEvents(List events) { } else { initializerWrapper.handleInputInitializerEvents(entry.getValue()); } - } } @@ -212,6 +211,7 @@ private VertexUpdateRegistrationHolder(String vertexName, Set stateSet; } @@ -261,7 +261,7 @@ private static class InputInitializerCallable implements private final AppContext appContext; InputInitializerCallable(InitializerWrapper initializer, UserGroupInformation ugi, - AppContext appContext) { + AppContext appContext) { this.initializerWrapper = initializer; this.ugi = ugi; this.appContext = appContext; @@ -298,7 +298,7 @@ private static class InputInitializerCallback implements private final TezVertexID vertexID; InputInitializerCallback(InitializerWrapper initializer, - EventHandler eventHandler, TezVertexID vertexID) { + EventHandler eventHandler, TezVertexID vertexID) { this.initializer = initializer; this.eventHandler = eventHandler; this.vertexID = vertexID; @@ -337,7 +337,6 @@ public void onFailure(Throwable t) { @InterfaceAudience.Private public static class InitializerWrapper implements VertexStateUpdateListener, TaskStateUpdateListener { - private final RootInputLeafOutput input; private final InputInitializer initializer; private final InputInitializerContext context; @@ -384,7 +383,7 @@ public void setComplete() { } public void registerForVertexStateUpdates(String vertexName, Set stateSet) { - synchronized(notificationRegisteredVertices) { + synchronized (notificationRegisteredVertices) { notificationRegisteredVertices.add(vertexName); } stateChangeNotifier.registerForVertexUpdates(vertexName, stateSet, this); @@ -395,7 +394,6 @@ private void unregisterForVertexStatusUpdates() { for (String vertexName : notificationRegisteredVertices) { stateChangeNotifier.unregisterForVertexUpdates(vertexName, this); } - } } @@ -414,7 +412,7 @@ public void onStateUpdated(VertexStateUpdate event) { } catch (Exception e) { appContext.getEventHandler().handle( new VertexEventRootInputFailed(vertexId, input.getName(), - new AMUserCodeException(Source.InputInitializer,e))); + new AMUserCodeException(Source.InputInitializer, e))); } } } @@ -522,7 +520,7 @@ private void sendEvents(List events) { } catch (Exception e) { appContext.getEventHandler().handle( new VertexEventRootInputFailed(vertexId, input.getName(), - new AMUserCodeException(Source.InputInitializer,e))); + new AMUserCodeException(Source.InputInitializer, e))); } } } @@ -533,5 +531,4 @@ private void unregisterForTaskStatusUpdates() { } } } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/StateChangeNotifier.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/StateChangeNotifier.java index 7d13f930c0..c79d6bb464 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/StateChangeNotifier.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/StateChangeNotifier.java @@ -18,7 +18,6 @@ package org.apache.tez.dag.app.dag; - import java.util.EnumSet; import java.util.List; import java.util.Set; @@ -53,7 +52,7 @@ public class StateChangeNotifier { private static final Logger LOG = LoggerFactory.getLogger(StateChangeNotifier.class); - + private final DAG dag; private final SetMultimap vertexListeners; private final ListMultimap lastKnowStatesMap; @@ -63,20 +62,20 @@ public class StateChangeNotifier { BlockingQueue eventQueue = new LinkedBlockingQueue(); private Thread eventHandlingThread; private volatile boolean stopEventHandling = false; - + private static class NotificationEvent { final VertexStateUpdate update; final VertexStateUpdateListener listener; - + public NotificationEvent(VertexStateUpdate update, VertexStateUpdateListener listener) { this.update = update; this.listener = listener; } - + void sendUpdate() { listener.onStateUpdated(update); } - + @Override public String toString() { return "[ VertexState:(" + update + ") Listener:" + listener + " ]"; @@ -90,7 +89,7 @@ public StateChangeNotifier(DAG dag) { this.lastKnowStatesMap = LinkedListMultimap.create(); startThread(); } - + private void startThread() { this.eventHandlingThread = new Thread("State Change Notifier DAG: " + dag.getID()) { @SuppressWarnings("unchecked") @@ -101,7 +100,7 @@ public void run() { try { event = eventQueue.take(); } catch (InterruptedException e) { - if(!stopEventHandling) { + if (!stopEventHandling) { LOG.warn("Continuing after interrupt : ", e); } continue; @@ -124,15 +123,15 @@ public void run() { this.eventHandlingThread.setDaemon(true); // dont block exit on this this.eventHandlingThread.start(); } - + @VisibleForTesting protected void processedEventFromQueue() { } - + @VisibleForTesting protected void addedEventToQueue() { } - + public void stop() { this.stopEventHandling = true; if (eventHandlingThread != null) @@ -208,7 +207,7 @@ private void enqueueNotification(NotificationEvent event) { LOG.error("Failed to put event", e); } } - + private final class ListenerContainer { final VertexStateUpdateListener listener; final Set states; @@ -264,8 +263,6 @@ public int hashCode() { private final ReentrantReadWriteLock.ReadLock taskReadLock = taskListenerLock.readLock(); private final ReentrantReadWriteLock.WriteLock taskWriteLock = taskListenerLock.writeLock(); - - public void registerForTaskSuccessUpdates(String vertexName, TaskStateUpdateListener listener) { TezVertexID vertexId = validateAndGetVertexId(vertexName); Objects.requireNonNull(listener, "listener cannot be null"); @@ -301,12 +298,10 @@ public void taskSucceeded(String vertexName, TezTaskID taskId, int attemptId) { // -------------- END OF TASK STATE CHANGE SECTION --------------- - private TezVertexID validateAndGetVertexId(String vertexName) { Objects.requireNonNull(vertexName, "VertexName cannot be null"); Vertex vertex = dag.getVertex(vertexName); Objects.requireNonNull(vertex, "Vertex does not exist: " + vertexName); return vertex.getVertexId(); } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java index cb1bc33866..370714f25d 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Task.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag; @@ -38,12 +38,19 @@ */ public interface Task extends TaskIDAware { TaskReport getReport(); + TaskState getState(); + TezCounters getCounters(); + float getProgress(); + Map getAttempts(); + TaskAttempt getAttempt(TezTaskAttemptID attemptID); + TaskAttempt getSuccessfulAttempt(); + /** Has Task reached the final state or not. */ boolean isFinished(); @@ -52,23 +59,23 @@ public interface Task extends TaskIDAware { * Can the output of the taskAttempt be committed. Note that once the task * gives a go for a commit, further canCommit requests from any other attempts * should return false. - * + * * @param taskAttemptID * @return whether the attempt's output can be committed or not. */ boolean canCommit(TezTaskAttemptID taskAttemptID); - + public Vertex getVertex(); - + public ArrayList getTaskAttemptTezEvents(TezTaskAttemptID attemptID, - int fromEventId, int maxEvents); - + int fromEventId, int maxEvents); + public List getDiagnostics(); public void registerTezEvent(TezEvent tezEvent); - + public TaskSpec getBaseTaskSpec(); - + public TaskLocationHint getTaskLocationHint(); long getFirstAttemptStartTime(); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java index 563e4c7192..1f488f0061 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttempt.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag; @@ -43,7 +43,7 @@ public static class TaskAttemptStatus { public TaskAttemptState state; public float progress; public TezCounters counters; - + public TaskAttemptStatus(TezTaskAttemptID id) { this.id = id; } @@ -51,6 +51,7 @@ public TaskAttemptStatus(TezTaskAttemptID id) { // insert these counters till they come natively from the task itself. // HDFS-5098 private AtomicBoolean localitySet = new AtomicBoolean(false); + public void setLocalityCounter(DAGCounter localityCounter) { if (!localitySet.get()) { localitySet.set(true); @@ -66,17 +67,24 @@ public void setLocalityCounter(DAGCounter localityCounter) { } Task getTask(); + TaskAttemptReport getReport(); + List getDiagnostics(); + TaskAttemptTerminationCause getTerminationCause(); + TezCounters getCounters(); + float getProgress(); + TaskAttemptState getState(); + TaskAttemptState getStateNoLock(); - + void setLastEventSent(TezEvent lastEventSent); - /** + /** * Has attempt reached the final state or not. * @return true if it has finished, else false */ @@ -86,7 +94,7 @@ public void setLocalityCounter(DAGCounter localityCounter) { * @return the container ID if a container is assigned, otherwise null. */ ContainerId getAssignedContainerID(); - + /** * @return the container if assigned, otherwise null */ @@ -96,23 +104,23 @@ public void setLocalityCounter(DAGCounter localityCounter) { * @return container mgr address if a container is assigned, otherwise null. */ String getAssignedContainerMgrAddress(); - + /** * @return node's id if a container is assigned, otherwise null. */ NodeId getNodeId(); - + /** * @return node's http address if a container is assigned, otherwise null. */ String getNodeHttpAddress(); - + /** * @return node's rack name if a container is assigned, otherwise null. */ String getNodeRackName(); - /** + /** * @return time at which container is launched. If container is not launched * yet, returns 0. */ @@ -124,7 +132,7 @@ public void setLocalityCounter(DAGCounter localityCounter) { */ long getScheduleTime(); - /** + /** * @return attempt's finish time. If attempt is not finished * yet, returns 0. */ diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java index 6ddfabb0d0..7fa7d56c03 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskAttemptStateInternal.java @@ -1,36 +1,36 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag; import org.apache.hadoop.classification.InterfaceAudience.Private; /** -* TaskAttemptImpl internal state machine states. -* -*/ + * TaskAttemptImpl internal state machine states. + * + */ @Private public enum TaskAttemptStateInternal { NEW, START_WAIT, SUBMITTED, RUNNING, - KILL_IN_PROGRESS, + KILL_IN_PROGRESS, FAIL_IN_PROGRESS, KILLED, FAILED, diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskStateInternal.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskStateInternal.java index 0e5a58100b..62631ac3e9 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskStateInternal.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskStateInternal.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskTerminationCause.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskTerminationCause.java index e2ae8dadbc..1f6cae9987 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskTerminationCause.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/TaskTerminationCause.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag; @@ -28,10 +28,10 @@ public enum TaskTerminationCause { /** Other vertex failed causing DAG to fail thus killing the parent vertex */ OTHER_VERTEX_FAILURE, - + /** One of the tasks for the source/destination vertex failed. */ - OTHER_TASK_FAILURE, - + OTHER_TASK_FAILURE, + /** One of the tasks of the destination vertex failed. */ OWN_TASK_FAILURE, @@ -39,6 +39,6 @@ public enum TaskTerminationCause { * throw Exception */ AM_USERCODE_FAILURE, - + TASK_RESCHEDULE_IN_COMMITTING, } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java index ff83e19a85..2e9fed245b 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/Vertex.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag; @@ -61,19 +61,23 @@ import org.apache.tez.runtime.api.impl.InputSpec; import org.apache.tez.runtime.api.impl.OutputSpec; - /** * Main interface to interact with the job. Provides only getters. */ public interface Vertex extends Comparable { TezVertexID getVertexId(); + public VertexPlan getVertexPlan(); int getDistanceFromRoot(); + LinkedHashMap getIOIndices(); + String getName(); + VertexState getState(); + /** * Get all the counters of this vertex. * @return aggregate task-counters @@ -93,88 +97,120 @@ public interface Vertex extends Comparable { void addCounters(TezCounters tezCounters); int getMaxTaskConcurrency(); + Map getTasks(); + Task getTask(TezTaskID taskID); + Task getTask(int taskIndex); + List getDiagnostics(); + int getTotalTasks(); + int getCompletedTasks(); + int getSucceededTasks(); + int getRunningTasks(); + float getProgress(); + float getCompletedTaskProgress(); + ProgressBuilder getVertexProgress(); + VertexStatusBuilder getVertexStatus(Set statusOptions); @Nullable TaskLocationHint getTaskLocationHint(TezTaskID taskID); void setParallelism(int parallelism, VertexLocationHint vertexLocationHint, - Map sourceEdgeManagers, - Map rootInputSpecUpdate, boolean fromVertexManager) + Map sourceEdgeManagers, + Map rootInputSpecUpdate, boolean fromVertexManager) throws AMUserCodeException; - + public void reconfigureVertex(int parallelism, - @Nullable VertexLocationHint locationHint, - @Nullable Map sourceEdgeProperties) throws AMUserCodeException; - + @Nullable VertexLocationHint locationHint, + @Nullable Map sourceEdgeProperties) throws AMUserCodeException; + public void reconfigureVertex(@Nullable Map rootInputSpecUpdate, - int parallelism, - @Nullable VertexLocationHint locationHint) throws AMUserCodeException; - + int parallelism, + @Nullable VertexLocationHint locationHint) throws AMUserCodeException; + public void reconfigureVertex(int parallelism, - @Nullable VertexLocationHint locationHint, - @Nullable Map sourceEdgeProperties, - @Nullable Map rootInputSpecUpdate) throws AMUserCodeException; + @Nullable VertexLocationHint locationHint, + @Nullable Map sourceEdgeProperties, + @Nullable Map rootInputSpecUpdate) throws AMUserCodeException; void setVertexLocationHint(VertexLocationHint vertexLocationHint); + void vertexReconfigurationPlanned(); + void doneReconfiguringVertex(); // CHANGE THESE TO LISTS AND MAINTAIN ORDER? void setInputVertices(Map inVertices); + void setOutputVertices(Map outVertices); VertexStatistics getStatistics(); - + Map getInputVertices(); + Map getOutputVertices(); - + Map getOutputCommitters(); void setAdditionalInputs(List inputs); + void setAdditionalOutputs(List outputs); @Nullable - public Map> - getAdditionalInputs(); + public Map> + getAdditionalInputs(); + @Nullable - public Map> - getAdditionalOutputs(); + public Map> + getAdditionalOutputs(); List getInputSpecList(int taskIndex) throws AMUserCodeException; + List getOutputSpecList(int taskIndex) throws AMUserCodeException; - + List getGroupInputSpecList(); + void addSharedOutputs(Set outputs); + Set getSharedOutputs(); int getInputVerticesCount(); + int getOutputVerticesCount(); + void scheduleTasks(List tasks); + void scheduleSpeculativeTask(TezTaskID taskId); + Resource getTaskResource(); - + public TaskAttemptEventInfo getTaskAttemptTezEvents(TezTaskAttemptID attemptID, - int fromEventId, int nextPreRoutedFromEventId, int maxEvents); - + int fromEventId, int nextPreRoutedFromEventId, int maxEvents); + void handleSpeculatorEvent(SpeculatorEvent event); + AbstractService getSpeculator(); + void initServices(); + void startServices(); + void stopServices(); + ProcessorDescriptor getProcessorDescriptor(); + public DAG getDAG(); + @Nullable VertexTerminationCause getTerminationCause(); @@ -197,39 +233,51 @@ public TaskAttemptEventInfo getTaskAttemptTezEvents(TezTaskAttemptID attemptID, public boolean isSpeculationEnabled(); public int getTaskSchedulerIdentifier(); + public int getContainerLauncherIdentifier(); + public int getTaskCommunicatorIdentifier(); public ServicePluginInfo getServicePluginInfo(); public long getInitTime(); + public long getStartTime(); + public long getFinishTime(); void reportTaskStartTime(long taskStartTime); + public long getFirstTaskStartTime(); + public long getLastTaskFinishTime(); VertexConfig getVertexConfig(); interface VertexConfig { int getMaxFailedTaskAttempts(); + int getMaxTaskAttempts(); + boolean getTaskRescheduleHigherPriority(); + boolean getTaskRescheduleRelaxedLocality(); /** * @return tez.task.max.allowed.output.failures. */ int getMaxAllowedOutputFailures(); + /** * @return tez.task.max.allowed.output.failures.fraction. */ double getMaxAllowedOutputFailuresFraction(); + /** * @return tez.am.max.allowed.time-sec.for-read-error. */ int getMaxAllowedTimeForTaskReadErrorSec(); + /** * @return tez.am.max.allowed.downstream.host.failures.fraction. */ @@ -237,6 +285,8 @@ interface VertexConfig { } void incrementRejectedTaskAttemptCount(); + int getRejectedTaskAttemptCount(); + Map> getDownstreamBlamingHosts(); } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexState.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexState.java index ba9d1aff0e..eea13ef477 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexState.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexState.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexTerminationCause.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexTerminationCause.java index 49be74df8d..f838812c68 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexTerminationCause.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/VertexTerminationCause.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag; @@ -31,7 +31,7 @@ public enum VertexTerminationCause { /** Initialization failed for one of the root Inputs */ ROOT_INPUT_INIT_FAILURE(VertexState.FAILED), - + /** This vertex failed as its AM usercode (VertexManager/EdgeManager/InputInitializer) * throw Exception */ @@ -54,10 +54,10 @@ public enum VertexTerminationCause { /** This vertex failed during init. */ INIT_FAILURE(VertexState.FAILED), - + INTERNAL_ERROR(VertexState.ERROR), - - /** error when writing recovery log */ + + /** error when writing recovery log */ RECOVERY_ERROR(VertexState.FAILED), /** This vertex failed due to counter limits exceeded. */ diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/CallableEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/CallableEvent.java index 7e6875293d..d22bc4f105 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/CallableEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/CallableEvent.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/CallableEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/CallableEventType.java index e9e93b9869..5ce7cc95b4 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/CallableEventType.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/CallableEventType.java @@ -1,25 +1,24 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; public enum CallableEventType { - - CALLABLE, + CALLABLE, } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGAppMasterEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGAppMasterEvent.java index b7cb3a48fc..3fddac5ab5 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGAppMasterEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGAppMasterEvent.java @@ -1,30 +1,28 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; import org.apache.tez.common.TezAbstractEvent; - public class DAGAppMasterEvent extends TezAbstractEvent { public DAGAppMasterEvent(DAGAppMasterEventType type) { super(type); } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGAppMasterEventDAGFinished.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGAppMasterEventDAGFinished.java index f58dace165..78fc3af823 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGAppMasterEventDAGFinished.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGAppMasterEventDAGFinished.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -26,7 +26,7 @@ public class DAGAppMasterEventDAGFinished extends DAGAppMasterEvent { private final DAGState dagState; public DAGAppMasterEventDAGFinished(TezDAGID dagId, - DAGState dagState) { + DAGState dagState) { super(DAGAppMasterEventType.DAG_FINISHED); this.dagId = dagId; this.dagState = dagState; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGAppMasterEventSchedulingServiceError.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGAppMasterEventSchedulingServiceError.java index cf49d20f41..c217b781b4 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGAppMasterEventSchedulingServiceError.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGAppMasterEventSchedulingServiceError.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -30,6 +30,6 @@ public DAGAppMasterEventSchedulingServiceError(String diagnostics) { @Override public String getDiagnosticInfo() { - return diagnostics; + return diagnostics; } } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGAppMasterEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGAppMasterEventType.java index 9cf2414e97..990d635cfe 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGAppMasterEventType.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGAppMasterEventType.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEvent.java index 4eed8ff984..97c9a183c4 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEvent.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -39,5 +39,4 @@ public DAGEvent(TezDAGID dagId, DAGEventType type) { public TezDAGID getDAGID() { return dagId; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCommitCompleted.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCommitCompleted.java index 2cfd60b9ce..700d9fff65 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCommitCompleted.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCommitCompleted.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,14 +20,14 @@ import org.apache.tez.dag.app.dag.impl.DAGImpl.OutputKey; import org.apache.tez.dag.records.TezDAGID; -public class DAGEventCommitCompleted extends DAGEvent{ +public class DAGEventCommitCompleted extends DAGEvent { private OutputKey outputKey; private boolean isSucceeded; private Throwable failException; public DAGEventCommitCompleted(TezDAGID dagId, OutputKey outputKey, - boolean isSucceeded, Throwable failException) { + boolean isSucceeded, Throwable failException) { super(dagId, DAGEventType.DAG_COMMIT_COMPLETED); this.outputKey = outputKey; this.isSucceeded = isSucceeded; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCounterUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCounterUpdate.java index da0724dd20..48d9669db3 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCounterUpdate.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventCounterUpdate.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,7 +26,7 @@ public class DAGEventCounterUpdate extends DAGEvent { List counterUpdates = null; - + public DAGEventCounterUpdate(TezDAGID dagId) { super(dagId, DAGEventType.DAG_COUNTER_UPDATE); counterUpdates = new ArrayList(); @@ -35,20 +35,20 @@ public DAGEventCounterUpdate(TezDAGID dagId) { public void addCounterUpdate(Enum key, long incrValue) { counterUpdates.add(new CounterIncrementalUpdate(key, incrValue)); } - + public List getCounterUpdates() { return counterUpdates; } - + public static class CounterIncrementalUpdate { Enum key; long incrValue; - + public CounterIncrementalUpdate(Enum key, long incrValue) { this.key = key; this.incrValue = incrValue; } - + public Enum getCounterKey() { return key; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventDiagnosticsUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventDiagnosticsUpdate.java index bde94a0bfa..cdd636b7b0 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventDiagnosticsUpdate.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventDiagnosticsUpdate.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventRecoverEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventRecoverEvent.java index 8e1edf0e43..fdee3051af 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventRecoverEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventRecoverEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -32,15 +32,15 @@ public DAGEventRecoverEvent(TezDAGID dagId, DAGState desiredState, DAGRecoveryDa this.desiredState = desiredState; this.recoveredDagData = recoveredDagData; } - + public DAGEventRecoverEvent(TezDAGID dagId, DAGRecoveryData recoveredDagData) { this(dagId, null, recoveredDagData); } - + public DAGState getDesiredState() { return desiredState; } - + public boolean hasDesiredState() { return this.desiredState != null; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventSchedulerUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventSchedulerUpdate.java index 1dedaecad9..c9d9b7d53f 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventSchedulerUpdate.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventSchedulerUpdate.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -28,21 +28,21 @@ public enum UpdateType { TA_SCHEDULE, TA_COMPLETED } - + private final TaskAttempt attempt; private final UpdateType updateType; - + public DAGEventSchedulerUpdate(UpdateType updateType, TaskAttempt attempt) { super(attempt.getDAGID(), - DAGEventType.DAG_SCHEDULER_UPDATE); + DAGEventType.DAG_SCHEDULER_UPDATE); this.attempt = attempt; this.updateType = updateType; } - + public UpdateType getUpdateType() { return updateType; } - + public TaskAttempt getAttempt() { return attempt; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventStartDag.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventStartDag.java index 9e6d75b019..a4cc65dd3c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventStartDag.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventStartDag.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -26,7 +26,7 @@ public class DAGEventStartDag extends DAGEvent { private final List additionalUrlsForClasspath; - + public DAGEventStartDag(TezDAGID dagId, List additionalUrlsForClasspath) { super(dagId, DAGEventType.DAG_START); this.additionalUrlsForClasspath = additionalUrlsForClasspath; @@ -35,4 +35,4 @@ public DAGEventStartDag(TezDAGID dagId, List additionalUrlsForClasspath) { public List getAdditionalUrlsForClasspath() { return this.additionalUrlsForClasspath; } -} \ No newline at end of file +} diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventType.java index bf3b30a0cf..56812f3436 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventType.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventType.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventVertexCompleted.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventVertexCompleted.java index 2c18be85dd..181a8bd4b9 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventVertexCompleted.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventVertexCompleted.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,7 +36,7 @@ public DAGEventVertexCompleted(TezVertexID vertexId, VertexState vertexState) { } public DAGEventVertexCompleted(TezVertexID vertexId, VertexState vertexState, - VertexTerminationCause terminationCause) { + VertexTerminationCause terminationCause) { super(vertexId.getDAGID(), DAGEventType.DAG_VERTEX_COMPLETED); this.vertexId = vertexId; this.vertexState = vertexState; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventVertexReRunning.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventVertexReRunning.java index 30e595906b..0fd0fa598e 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventVertexReRunning.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DAGEventVertexReRunning.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -32,6 +32,4 @@ public DAGEventVertexReRunning(TezVertexID vertexId) { public TezVertexID getVertexId() { return vertexId; } - } - diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DiagnosableEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DiagnosableEvent.java index 4187309d14..0156a5fecb 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DiagnosableEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/DiagnosableEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/RecoveryEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/RecoveryEvent.java index cad38241fb..f9de6756ed 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/RecoveryEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/RecoveryEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/SpeculatorEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/SpeculatorEvent.java index 3863a2a6f4..1e0aa6bc84 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/SpeculatorEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/SpeculatorEvent.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -23,14 +23,13 @@ public class SpeculatorEvent extends TezAbstractEvent { private final TezVertexID vertexId; - + public SpeculatorEvent(SpeculatorEventType type, TezVertexID vertexId) { super(type); this.vertexId = vertexId; } - + public TezVertexID getVertexId() { return vertexId; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/SpeculatorEventTaskAttemptStatusUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/SpeculatorEventTaskAttemptStatusUpdate.java index 7ab6141c39..91a5db67cd 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/SpeculatorEventTaskAttemptStatusUpdate.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/SpeculatorEventTaskAttemptStatusUpdate.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -26,14 +26,14 @@ public class SpeculatorEventTaskAttemptStatusUpdate extends SpeculatorEvent { final TaskAttemptState state; final long timestamp; final boolean justStarted; - + public SpeculatorEventTaskAttemptStatusUpdate(TezTaskAttemptID taId, TaskAttemptState state, - long timestamp) { + long timestamp) { this(taId, state, timestamp, false); } - + public SpeculatorEventTaskAttemptStatusUpdate(TezTaskAttemptID taId, TaskAttemptState state, - long timestamp, boolean justStarted) { + long timestamp, boolean justStarted) { super(SpeculatorEventType.S_TASK_ATTEMPT_STATUS_UPDATE, taId.getVertexID()); this.id = taId; this.state = state; @@ -44,17 +44,16 @@ public SpeculatorEventTaskAttemptStatusUpdate(TezTaskAttemptID taId, TaskAttempt public long getTimestamp() { return timestamp; } - + public TezTaskAttemptID getAttemptId() { return id; } - + public boolean hasJustStarted() { return justStarted; } - + public TaskAttemptState getTaskAttemptState() { return state; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/SpeculatorEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/SpeculatorEventType.java index f3b5c10503..b63620a5bf 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/SpeculatorEventType.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/SpeculatorEventType.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -25,5 +25,4 @@ public enum SpeculatorEventType { //Producer:TaskAttempt S_TASK_ATTEMPT_STATUS_UPDATE, - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEvent.java index d5d8481a54..2fc717ce1c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEvent.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -29,7 +29,7 @@ public class TaskAttemptEvent extends TezAbstractEvent implements TaskAttemptIDAware { private TezTaskAttemptID attemptID; - + /** * Create a new TaskAttemptEvent. * @param id the id of the task attempt @@ -44,7 +44,7 @@ public TaskAttemptEvent(TezTaskAttemptID id, TaskAttemptEventType type) { public TezTaskAttemptID getTaskAttemptID() { return attemptID; } - + @Override public int getSerializingHash() { return attemptID.getTaskID().getSerializingHash(); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventAttemptFailed.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventAttemptFailed.java index d4c7273c97..ad8e3883be 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventAttemptFailed.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventAttemptFailed.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -24,8 +24,8 @@ import org.apache.tez.dag.records.TezTaskAttemptID; import org.apache.tez.runtime.api.TaskFailureType; -public class TaskAttemptEventAttemptFailed extends TaskAttemptEvent - implements DiagnosableEvent, TaskAttemptEventTerminationCauseEvent, RecoveryEvent { +public class TaskAttemptEventAttemptFailed extends TaskAttemptEvent + implements DiagnosableEvent, TaskAttemptEventTerminationCauseEvent, RecoveryEvent { private final String diagnostics; private final TaskAttemptTerminationCause errorCause; @@ -56,7 +56,7 @@ public TaskAttemptEventAttemptFailed(TezTaskAttemptID id, public String getDiagnosticInfo() { return diagnostics; } - + @Override public TaskAttemptTerminationCause getTerminationCause() { return errorCause; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventAttemptKilled.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventAttemptKilled.java index 4642443848..858590e1c4 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventAttemptKilled.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventAttemptKilled.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -37,13 +37,13 @@ public TaskAttemptEventAttemptKilled(TezTaskAttemptID id, } public TaskAttemptEventAttemptKilled(TezTaskAttemptID id, - String diagnostics, - TaskAttemptTerminationCause errorCause, - boolean fromRecovery) { + String diagnostics, + TaskAttemptTerminationCause errorCause, + boolean fromRecovery) { this(id, diagnostics, errorCause); this.fromRecovery = fromRecovery; } - + @Override public String getDiagnosticInfo() { return diagnostics; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminated.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminated.java index 3db2ffcbb8..6bcc169050 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminated.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminated.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -29,7 +29,7 @@ public class TaskAttemptEventContainerTerminated extends TaskAttemptEvent private final TaskAttemptTerminationCause errorCause; public TaskAttemptEventContainerTerminated(ContainerId containerId, TezTaskAttemptID id, - String message, TaskAttemptTerminationCause errCause) { + String message, TaskAttemptTerminationCause errCause) { super(id, TaskAttemptEventType.TA_CONTAINER_TERMINATED); this.containerId = containerId; this.message = message; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminatedBySystem.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminatedBySystem.java index 4efbf88948..2e7a4f9681 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminatedBySystem.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminatedBySystem.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -22,15 +22,15 @@ import org.apache.tez.dag.records.TaskAttemptTerminationCause; import org.apache.tez.dag.records.TezTaskAttemptID; -public class TaskAttemptEventContainerTerminatedBySystem extends TaskAttemptEvent - implements DiagnosableEvent, TaskAttemptEventTerminationCauseEvent { +public class TaskAttemptEventContainerTerminatedBySystem extends TaskAttemptEvent + implements DiagnosableEvent, TaskAttemptEventTerminationCauseEvent { private final ContainerId containerId; private final String diagnostics; private final TaskAttemptTerminationCause errorCause; public TaskAttemptEventContainerTerminatedBySystem(ContainerId containerId, TezTaskAttemptID id, - String diagnostics, TaskAttemptTerminationCause errorCause) { + String diagnostics, TaskAttemptTerminationCause errorCause) { super(id, TaskAttemptEventType.TA_CONTAINER_TERMINATED_BY_SYSTEM); this.containerId = containerId; this.diagnostics = diagnostics; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java index 02d04a5dcc..4cfc74df97 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventContainerTerminating.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -27,7 +27,7 @@ public class TaskAttemptEventContainerTerminating extends TaskAttemptEvent private final TaskAttemptTerminationCause errorCause; public TaskAttemptEventContainerTerminating(TezTaskAttemptID id, - String diagMessage, TaskAttemptTerminationCause errCause) { + String diagMessage, TaskAttemptTerminationCause errCause) { super(id, TaskAttemptEventType.TA_CONTAINER_TERMINATING); this.message = diagMessage; this.errorCause = errCause; @@ -42,5 +42,4 @@ public String getDiagnosticInfo() { public TaskAttemptTerminationCause getTerminationCause() { return errorCause; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventKillRequest.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventKillRequest.java index 96cf0e6bdf..57d7c09c43 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventKillRequest.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventKillRequest.java @@ -1,27 +1,27 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; import org.apache.tez.dag.records.TaskAttemptTerminationCause; import org.apache.tez.dag.records.TezTaskAttemptID; -public class TaskAttemptEventKillRequest extends TaskAttemptEvent - implements DiagnosableEvent, TaskAttemptEventTerminationCauseEvent, RecoveryEvent { +public class TaskAttemptEventKillRequest extends TaskAttemptEvent + implements DiagnosableEvent, TaskAttemptEventTerminationCauseEvent, RecoveryEvent { private final String message; private final TaskAttemptTerminationCause errorCause; @@ -34,7 +34,7 @@ public TaskAttemptEventKillRequest(TezTaskAttemptID id, String message, TaskAtte } public TaskAttemptEventKillRequest(TezTaskAttemptID id, String message, TaskAttemptTerminationCause err, - boolean fromRecovery) { + boolean fromRecovery) { this(id, message, err); this.fromRecovery = fromRecovery; } @@ -53,5 +53,4 @@ public TaskAttemptTerminationCause getTerminationCause() { public boolean isFromRecovery() { return fromRecovery; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java index 541ef00418..835853d9e6 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventNodeFailed.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -20,14 +20,14 @@ import org.apache.tez.dag.records.TaskAttemptTerminationCause; import org.apache.tez.dag.records.TezTaskAttemptID; -public class TaskAttemptEventNodeFailed extends TaskAttemptEvent - implements DiagnosableEvent, TaskAttemptEventTerminationCauseEvent { +public class TaskAttemptEventNodeFailed extends TaskAttemptEvent + implements DiagnosableEvent, TaskAttemptEventTerminationCauseEvent { private final String message; private final TaskAttemptTerminationCause errorCause; public TaskAttemptEventNodeFailed(TezTaskAttemptID id, - String diagMessage, TaskAttemptTerminationCause errorCause) { + String diagMessage, TaskAttemptTerminationCause errorCause) { super(id, TaskAttemptEventType.TA_NODE_FAILED); this.message = diagMessage; this.errorCause = errorCause; @@ -42,5 +42,4 @@ public String getDiagnosticInfo() { public TaskAttemptTerminationCause getTerminationCause() { return errorCause; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventOutputFailed.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventOutputFailed.java index fbdd2305c7..062c6061e0 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventOutputFailed.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventOutputFailed.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -22,23 +22,23 @@ import org.apache.tez.dag.records.TezTaskAttemptID; import org.apache.tez.runtime.api.impl.TezEvent; -public class TaskAttemptEventOutputFailed extends TaskAttemptEvent - implements TaskAttemptEventTerminationCauseEvent { - +public class TaskAttemptEventOutputFailed extends TaskAttemptEvent + implements TaskAttemptEventTerminationCauseEvent { + private TezEvent inputFailedEvent; private int consumerTaskNumber; - + public TaskAttemptEventOutputFailed(TezTaskAttemptID sourceTaskAttemptId, - TezEvent tezEvent, int numConsumers) { + TezEvent tezEvent, int numConsumers) { super(sourceTaskAttemptId, TaskAttemptEventType.TA_OUTPUT_FAILED); this.inputFailedEvent = tezEvent; this.consumerTaskNumber = numConsumers; } - + public TezEvent getInputFailedEvent() { return inputFailedEvent; } - + public int getConsumerTaskNumber() { return consumerTaskNumber; } @@ -47,5 +47,4 @@ public int getConsumerTaskNumber() { public TaskAttemptTerminationCause getTerminationCause() { return TaskAttemptTerminationCause.OUTPUT_LOST; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventSchedule.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventSchedule.java index 5c02db9a83..bebd8d82a6 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventSchedule.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventSchedule.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -33,9 +33,8 @@ public TaskAttemptEventSchedule(TezTaskAttemptID id, int lowLimit, int highLimit public int getPriorityLowLimit() { return priLowLimit; } - + public int getPriorityHighLimit() { return priHighLimit; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStartedRemotely.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStartedRemotely.java index d83eda97ea..d0d95df22b 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStartedRemotely.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStartedRemotely.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -30,4 +30,4 @@ public TaskAttemptEventStartedRemotely(TezTaskAttemptID id) { public boolean isFromRecovery() { return false; } -} \ No newline at end of file +} diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java index 458679c74b..28f905e95f 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventStatusUpdate.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -22,24 +22,24 @@ import org.apache.tez.runtime.api.events.TaskStatusUpdateEvent; public class TaskAttemptEventStatusUpdate extends TaskAttemptEvent { - + private TaskStatusUpdateEvent taskAttemptStatus; private boolean readErrorReported = false; - + public TaskAttemptEventStatusUpdate(TezTaskAttemptID id, - TaskStatusUpdateEvent statusEvent) { + TaskStatusUpdateEvent statusEvent) { super(id, TaskAttemptEventType.TA_STATUS_UPDATE); this.taskAttemptStatus = statusEvent; } - + public TaskStatusUpdateEvent getStatusEvent() { return this.taskAttemptStatus; } - + public void setReadErrorReported(boolean value) { readErrorReported = value; } - + public boolean getReadErrorReported() { return readErrorReported; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventSubmitted.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventSubmitted.java index 9fae77261f..c997e96238 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventSubmitted.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventSubmitted.java @@ -18,7 +18,6 @@ package org.apache.tez.dag.app.dag.event; - import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.tez.dag.records.TezTaskAttemptID; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventTerminationCauseEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventTerminationCauseEvent.java index 70c20e302c..4ae05baf1e 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventTerminationCauseEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventTerminationCauseEvent.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventTezEventUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventTezEventUpdate.java index bef92485ff..dee9c67796 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventTezEventUpdate.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventTezEventUpdate.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventType.java index 63779fd948..3cf5131571 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventType.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskAttemptEventType.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -35,7 +35,7 @@ public enum TaskAttemptEventType { TA_FAILED, TA_KILLED, // Generated by TaskCommunicators TA_TIMED_OUT, - + //Producer: Client, Scheduler, On speculation. TA_KILL_REQUEST, @@ -54,8 +54,7 @@ public enum TaskAttemptEventType { // The node running the task attempt failed. TA_NODE_FAILED, - + // Producer: consumer destination vertex TA_OUTPUT_FAILED, - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEvent.java index 9e741388fa..8889c2d86f 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEvent.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -39,7 +39,7 @@ public TaskEvent(TezTaskID taskId, TaskEventType type) { public TezTaskID getTaskID() { return taskId; } - + @Override public int getSerializingHash() { return taskId.getSerializingHash(); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventScheduleTask.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventScheduleTask.java index 70d6043f40..8ef9682ea5 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventScheduleTask.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventScheduleTask.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -28,17 +28,17 @@ public class TaskEventScheduleTask extends TaskEvent implements RecoveryEvent { private final boolean fromRecovery; public TaskEventScheduleTask(TezTaskID taskId, TaskSpec baseTaskSpec, TaskLocationHint locationHint, - boolean fromRecovery) { + boolean fromRecovery) { super(taskId, TaskEventType.T_SCHEDULE); this.baseTaskSpec = baseTaskSpec; this.locationHint = locationHint; this.fromRecovery = fromRecovery; } - + public TaskSpec getBaseTaskSpec() { return baseTaskSpec; } - + public TaskLocationHint getTaskLocationHint() { return locationHint; } @@ -47,4 +47,4 @@ public TaskLocationHint getTaskLocationHint() { public boolean isFromRecovery() { return fromRecovery; } -} \ No newline at end of file +} diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTAFailed.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTAFailed.java index f68549dd6b..5fe84a9d4c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTAFailed.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTAFailed.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTAKilled.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTAKilled.java index c410361493..d0df9d5905 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTAKilled.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTAKilled.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTALaunched.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTALaunched.java index a9d46c53c6..946f847b46 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTALaunched.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTALaunched.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTASucceeded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTASucceeded.java index e74e2bf132..13b5730649 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTASucceeded.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTASucceeded.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTAUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTAUpdate.java index 346c9d13af..41c2a3aef8 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTAUpdate.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTAUpdate.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTermination.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTermination.java index 1605869ed1..0e1be5bb67 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTermination.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventTermination.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -39,7 +39,7 @@ public TaskEventTermination(TezTaskID taskID, TaskAttemptTerminationCause errorC } public TaskEventTermination(TezTaskID taskID, TaskAttemptTerminationCause errorCause, String diagnostics, - boolean fromRecovery) { + boolean fromRecovery) { this(taskID, errorCause, diagnostics); this.fromRecovery = fromRecovery; } @@ -58,5 +58,4 @@ public TaskAttemptTerminationCause getTerminationCause() { public boolean isFromRecovery() { return fromRecovery; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventType.java index 726e13e23f..4387bf8111 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventType.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/TaskEventType.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -25,7 +25,7 @@ public enum TaskEventType { //Producer:Client, Job T_TERMINATE, - + //Producer:Job T_SCHEDULE, @@ -37,5 +37,4 @@ public enum TaskEventType { T_ATTEMPT_FAILED, T_ATTEMPT_SUCCEEDED, T_ATTEMPT_KILLED, - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEvent.java index 6957a50dc7..b914a9a5cb 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEvent.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventCommitCompleted.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventCommitCompleted.java index 77272f2f70..326ec15a2f 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventCommitCompleted.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventCommitCompleted.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,7 +26,7 @@ public class VertexEventCommitCompleted extends VertexEvent { private Throwable failException; public VertexEventCommitCompleted(TezVertexID vertexId, String outputName, - boolean isSucceeded, Throwable failException) { + boolean isSucceeded, Throwable failException) { super(vertexId, VertexEventType.V_COMMIT_COMPLETED); this.outputName = outputName; this.isSucceeded = isSucceeded; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventInputDataInformation.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventInputDataInformation.java index 6b5cad5ab7..561c983d42 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventInputDataInformation.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventInputDataInformation.java @@ -1,21 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -27,14 +26,13 @@ public class VertexEventInputDataInformation extends VertexEvent { private final List events; - + public VertexEventInputDataInformation(TezVertexID vertexId, List events) { super(vertexId, VertexEventType.V_INPUT_DATA_INFORMATION); this.events = events; } - + public List getEvents() { return events; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventManagerUserCodeError.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventManagerUserCodeError.java index 022620a064..041be03c15 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventManagerUserCodeError.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventManagerUserCodeError.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -23,14 +23,13 @@ public class VertexEventManagerUserCodeError extends VertexEvent { final AMUserCodeException e; - + public VertexEventManagerUserCodeError(TezVertexID vertexId, AMUserCodeException e) { super(vertexId, VertexEventType.V_MANAGER_USER_CODE_ERROR); this.e = e; } - + public AMUserCodeException getError() { return e; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventNullEdgeInitialized.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventNullEdgeInitialized.java index bedf28b4fd..fe42245047 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventNullEdgeInitialized.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventNullEdgeInitialized.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -25,17 +25,17 @@ public class VertexEventNullEdgeInitialized extends VertexEvent { final Edge edge; final Vertex vertex; - + public VertexEventNullEdgeInitialized(TezVertexID vertexId, Edge edge, Vertex vertex) { super(vertexId, VertexEventType.V_NULL_EDGE_INITIALIZED); this.edge = edge; this.vertex = vertex; } - + public Edge getEdge() { return edge; } - + public Vertex getVertex() { return vertex; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRecoverVertex.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRecoverVertex.java index 4203689a02..7ec66a459b 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRecoverVertex.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRecoverVertex.java @@ -1,19 +1,19 @@ /* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRootInputFailed.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRootInputFailed.java index b7701d7b03..9e608df5ec 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRootInputFailed.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRootInputFailed.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -22,7 +22,7 @@ import org.apache.tez.dag.records.TezVertexID; public class VertexEventRootInputFailed extends VertexEvent { - + private final String inputName; private final AMUserCodeException error; @@ -31,11 +31,11 @@ public VertexEventRootInputFailed(TezVertexID vertexId, String inputName, AMUser this.inputName = inputName; this.error = error; } - + public AMUserCodeException getError() { return this.error; } - + public String getInputName() { return this.inputName; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRootInputInitialized.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRootInputInitialized.java index 3c5e0be9cf..57343fa4c2 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRootInputInitialized.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRootInputInitialized.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -38,13 +38,12 @@ public VertexEventRootInputInitialized(TezVertexID vertexId, String inputName, L this.events = events; } } - + public List getEvents() { return events; } - + public String getInputName() { return this.inputName; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRouteEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRouteEvent.java index 211202ded2..16d18a35f4 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRouteEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventRouteEvent.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -24,7 +24,7 @@ import org.apache.tez.runtime.api.impl.TezEvent; public class VertexEventRouteEvent extends VertexEvent { - + final List events; public VertexEventRouteEvent(TezVertexID vertexId, List events) { @@ -35,5 +35,4 @@ public VertexEventRouteEvent(TezVertexID vertexId, List events) { public List getEvents() { return events; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceTaskAttemptCompleted.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceTaskAttemptCompleted.java index 6cd38a9b29..788660c76c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceTaskAttemptCompleted.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceTaskAttemptCompleted.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -27,7 +27,7 @@ public class VertexEventSourceTaskAttemptCompleted extends VertexEvent { public VertexEventSourceTaskAttemptCompleted( TezVertexID targetVertexId, VertexEventTaskAttemptCompleted completionEvent) { - super(targetVertexId, + super(targetVertexId, VertexEventType.V_SOURCE_TASK_ATTEMPT_COMPLETED); this.completionEvent = completionEvent; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceVertexStarted.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceVertexStarted.java index 5cc9c238f3..fae86c781f 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceVertexStarted.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventSourceVertexStarted.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -24,19 +24,19 @@ public class VertexEventSourceVertexStarted extends VertexEvent { final int sourceDistanceFromRoot; final TezVertexID sourceVertexId; - + public VertexEventSourceVertexStarted(TezVertexID vertexId, - TezVertexID sourceVertexId, - int distanceFromRoot) { + TezVertexID sourceVertexId, + int distanceFromRoot) { super(vertexId, VertexEventType.V_SOURCE_VERTEX_STARTED); this.sourceDistanceFromRoot = distanceFromRoot; this.sourceVertexId = sourceVertexId; } - + public int getSourceDistanceFromRoot() { return sourceDistanceFromRoot; } - + public TezVertexID getSourceVertexId() { return sourceVertexId; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskAttemptCompleted.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskAttemptCompleted.java index e23d27cf6e..6c8edafe13 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskAttemptCompleted.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskAttemptCompleted.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -25,9 +25,9 @@ public class VertexEventTaskAttemptCompleted extends VertexEvent { private TezTaskAttemptID attemptId; private TaskAttemptStateInternal attempState; - + public VertexEventTaskAttemptCompleted(TezTaskAttemptID taskAttemptId, - TaskAttemptStateInternal state) { + TaskAttemptStateInternal state) { super(taskAttemptId.getVertexID(), VertexEventType.V_TASK_ATTEMPT_COMPLETED); this.attemptId = taskAttemptId; @@ -37,7 +37,7 @@ public VertexEventTaskAttemptCompleted(TezTaskAttemptID taskAttemptId, public TezTaskAttemptID getTaskAttemptId() { return attemptId; } - + public TaskAttemptStateInternal getTaskAttemptState() { return attempState; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskCompleted.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskCompleted.java index 4bf4e2440e..f70d7216f0 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskCompleted.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskCompleted.java @@ -1,27 +1,26 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; import org.apache.tez.dag.api.oldrecords.TaskState; import org.apache.tez.dag.records.TezTaskID; - public class VertexEventTaskCompleted extends VertexEvent { private TezTaskID taskID; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskReschedule.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskReschedule.java index 9706246dad..4454a0f627 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskReschedule.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTaskReschedule.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; import org.apache.tez.dag.records.TezTaskID; @@ -30,5 +30,4 @@ public VertexEventTaskReschedule(TezTaskID taskID) { public TezTaskID getTaskID() { return taskID; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTermination.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTermination.java index 647f6003d8..34721602ab 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTermination.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventTermination.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -23,7 +23,7 @@ /** * Specialized VertexEvent for termination conditions. - * + * * The VertexEventType is always V_TERMINATE. * The terminationCause denotes the specific reason why the vertex is being terminated. */ @@ -39,5 +39,4 @@ public VertexEventTermination(TezVertexID vertexId, VertexTerminationCause termi public VertexTerminationCause getTerminationCause() { return terminationCause; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventType.java index ed325290cb..4c0350ad75 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventType.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexEventType.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.event; @@ -28,41 +28,40 @@ public enum VertexEventType { //Producer:Job V_INIT, - + //Producer:Vertex V_COMPLETED, V_START, V_SOURCE_TASK_ATTEMPT_COMPLETED, V_SOURCE_VERTEX_STARTED, V_DELETE_SHUFFLE_DATA, - + //Producer:Task V_TASK_COMPLETED, V_TASK_RESCHEDULED, V_TASK_ATTEMPT_COMPLETED, - + //Producer:Any component V_INTERNAL_ERROR, V_MANAGER_USER_CODE_ERROR, - + V_ROUTE_EVENT, - + //Producer: VertexInputInitializer V_ROOT_INPUT_INITIALIZED, V_ROOT_INPUT_FAILED, - + V_INPUT_DATA_INFORMATION, // Recover Event, Producer:DAG V_RECOVER, - + // Producer: Vertex V_READY_TO_INIT, - + // Producer: Edge V_NULL_EDGE_INITIALIZED, // Committer V_COMMIT_COMPLETED, - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexShuffleDataDeletion.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexShuffleDataDeletion.java index 8ea3a154b3..e30fe736ea 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexShuffleDataDeletion.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/VertexShuffleDataDeletion.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,7 +20,6 @@ import org.apache.tez.dag.app.dag.Vertex; - public class VertexShuffleDataDeletion extends VertexEvent { // child vertex private Vertex sourceVertex; @@ -40,4 +39,4 @@ public Vertex getSourceVertex() { public Vertex getTargetVertex() { return targetVertex; } -} \ No newline at end of file +} diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/package-info.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/package-info.java index 07369228eb..28439c9b1a 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/package-info.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/event/package-info.java @@ -17,4 +17,5 @@ */ @InterfaceAudience.Private package org.apache.tez.dag.app.dag.event; + import org.apache.hadoop.classification.InterfaceAudience; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/AMUserCodeException.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/AMUserCodeException.java index 22b1211544..787d4eb47e 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/AMUserCodeException.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/AMUserCodeException.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,7 +21,6 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.tez.dag.api.TezException; - /** * Wrap the exception of user code in AM. Now we only have 3 kinds of user code in AM: *

  • VertexManager
    • @@ -38,9 +37,9 @@ public static enum Source { EdgeManager, InputInitializer } - + private Source source; - + public AMUserCodeException(Source source, String message, Throwable cause) { super(message, cause); this.source = source; @@ -50,7 +49,7 @@ public AMUserCodeException(Source source, Throwable cause) { super(cause); this.source = source; } - + public Source getSource() { return source; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BroadcastEdgeManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BroadcastEdgeManager.java index 867403fcd5..bd3cd77f57 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BroadcastEdgeManager.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/BroadcastEdgeManager.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.impl; @@ -38,44 +38,44 @@ public BroadcastEdgeManager(EdgeManagerPluginContext context) { @Override public void initialize() { } - + @Override public int getNumDestinationTaskPhysicalInputs(int destinationTaskIndex) { return getContext().getSourceVertexNumTasks(); } - + @Override public int getNumSourceTaskPhysicalOutputs(int sourceTaskIndex) { return 1; } - + @Override public void routeDataMovementEventToDestination(DataMovementEvent event, - int sourceTaskIndex, int sourceOutputIndex, - Map> destinationTaskAndInputIndices) { + int sourceTaskIndex, int sourceOutputIndex, + Map> destinationTaskAndInputIndices) { List inputIndices = Collections.singletonList(sourceTaskIndex); // for each task make the i-th source task as the i-th physical input - for (int i=0; i> destinationTaskAndInputIndices) { + Map> destinationTaskAndInputIndices) { List inputIndices = Collections.singletonList(sourceTaskIndex); // for each task make the i-th source task as the i-th physical input - for (int i=0; i + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.impl; @@ -154,9 +154,9 @@ /** Implementation of Job interface. Maintains the state machines of Job. * The read and write calls use ReadWriteLock for concurrency. */ -@SuppressWarnings({ "rawtypes", "unchecked" }) +@SuppressWarnings({"rawtypes", "unchecked"}) public class DAGImpl implements org.apache.tez.dag.app.dag.DAG, - EventHandler { + EventHandler { private static final Logger LOG = LoggerFactory.getLogger(DAGImpl.class); private static final String LINE_SEPARATOR = System @@ -221,7 +221,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG, private final AtomicBoolean internalErrorTriggered = new AtomicBoolean(false); Map localResources; - + long startDAGCpuTime = 0; long startDAGGCTime = 0; @@ -234,7 +234,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG, @VisibleForTesting Map> commitFutures - = new HashMap>(); + = new HashMap>(); private static final DiagnosticsUpdateTransition DIAGNOSTIC_UPDATE_TRANSITION = new DiagnosticsUpdateTransition(); @@ -243,211 +243,211 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG, private static final CounterUpdateTransition COUNTER_UPDATE_TRANSITION = new CounterUpdateTransition(); private static final DAGSchedulerUpdateTransition - DAG_SCHEDULER_UPDATE_TRANSITION = new DAGSchedulerUpdateTransition(); + DAG_SCHEDULER_UPDATE_TRANSITION = new DAGSchedulerUpdateTransition(); private static final CommitCompletedTransition COMMIT_COMPLETED_TRANSITION = new CommitCompletedTransition(); private final MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean(); protected static final - StateMachineFactory - stateMachineFactory - = new StateMachineFactory - (DAGState.NEW) - - // Transitions from NEW state - .addTransition(DAGState.NEW, DAGState.NEW, - DAGEventType.DAG_DIAGNOSTIC_UPDATE, - DIAGNOSTIC_UPDATE_TRANSITION) - // either recovered to FINISHED state or recovered to NEW to rerun the dag based on the recovery data - .addTransition(DAGState.NEW, - EnumSet.of(DAGState.NEW, DAGState.SUCCEEDED, - DAGState.FAILED, DAGState.KILLED, - DAGState.ERROR), - DAGEventType.DAG_RECOVER, - new RecoverTransition()) - .addTransition(DAGState.NEW, DAGState.NEW, - DAGEventType.DAG_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) - .addTransition - (DAGState.NEW, + StateMachineFactory + stateMachineFactory + = new StateMachineFactory + (DAGState.NEW) + + // Transitions from NEW state + .addTransition(DAGState.NEW, DAGState.NEW, + DAGEventType.DAG_DIAGNOSTIC_UPDATE, + DIAGNOSTIC_UPDATE_TRANSITION) + // either recovered to FINISHED state or recovered to NEW to rerun the dag based on the recovery data + .addTransition(DAGState.NEW, + EnumSet.of(DAGState.NEW, DAGState.SUCCEEDED, + DAGState.FAILED, DAGState.KILLED, + DAGState.ERROR), + DAGEventType.DAG_RECOVER, + new RecoverTransition()) + .addTransition(DAGState.NEW, DAGState.NEW, + DAGEventType.DAG_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) + .addTransition + (DAGState.NEW, EnumSet.of(DAGState.INITED, DAGState.FAILED), DAGEventType.DAG_INIT, new InitTransition()) - .addTransition(DAGState.NEW, EnumSet.of(DAGState.KILLED, DAGState.FAILED), - DAGEventType.DAG_TERMINATE, - new KillNewJobTransition()) - .addTransition(DAGState.NEW, DAGState.ERROR, - DAGEventType.INTERNAL_ERROR, - INTERNAL_ERROR_TRANSITION) - - // Transitions from INITED state - .addTransition(DAGState.INITED, DAGState.INITED, - DAGEventType.DAG_DIAGNOSTIC_UPDATE, - DIAGNOSTIC_UPDATE_TRANSITION) - .addTransition(DAGState.INITED, DAGState.INITED, - DAGEventType.DAG_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) - .addTransition(DAGState.INITED, DAGState.RUNNING, - DAGEventType.DAG_START, - new StartTransition()) - .addTransition(DAGState.INITED, EnumSet.of(DAGState.KILLED, DAGState.FAILED), - DAGEventType.DAG_TERMINATE, - new KillInitedJobTransition()) - .addTransition(DAGState.INITED, DAGState.ERROR, - DAGEventType.INTERNAL_ERROR, - INTERNAL_ERROR_TRANSITION) - - // Transitions from RUNNING state - .addTransition - (DAGState.RUNNING, + .addTransition(DAGState.NEW, EnumSet.of(DAGState.KILLED, DAGState.FAILED), + DAGEventType.DAG_TERMINATE, + new KillNewJobTransition()) + .addTransition(DAGState.NEW, DAGState.ERROR, + DAGEventType.INTERNAL_ERROR, + INTERNAL_ERROR_TRANSITION) + + // Transitions from INITED state + .addTransition(DAGState.INITED, DAGState.INITED, + DAGEventType.DAG_DIAGNOSTIC_UPDATE, + DIAGNOSTIC_UPDATE_TRANSITION) + .addTransition(DAGState.INITED, DAGState.INITED, + DAGEventType.DAG_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) + .addTransition(DAGState.INITED, DAGState.RUNNING, + DAGEventType.DAG_START, + new StartTransition()) + .addTransition(DAGState.INITED, EnumSet.of(DAGState.KILLED, DAGState.FAILED), + DAGEventType.DAG_TERMINATE, + new KillInitedJobTransition()) + .addTransition(DAGState.INITED, DAGState.ERROR, + DAGEventType.INTERNAL_ERROR, + INTERNAL_ERROR_TRANSITION) + + // Transitions from RUNNING state + .addTransition + (DAGState.RUNNING, EnumSet.of(DAGState.RUNNING, DAGState.COMMITTING, - DAGState.SUCCEEDED, DAGState.TERMINATING,DAGState.FAILED), + DAGState.SUCCEEDED, DAGState.TERMINATING, DAGState.FAILED), DAGEventType.DAG_VERTEX_COMPLETED, new VertexCompletedTransition()) - .addTransition(DAGState.RUNNING, EnumSet.of(DAGState.RUNNING, DAGState.TERMINATING), + .addTransition(DAGState.RUNNING, EnumSet.of(DAGState.RUNNING, DAGState.TERMINATING), + DAGEventType.DAG_VERTEX_RERUNNING, + new VertexReRunningTransition()) + .addTransition(DAGState.RUNNING, DAGState.TERMINATING, + DAGEventType.DAG_TERMINATE, new DAGKilledTransition()) + .addTransition(DAGState.RUNNING, DAGState.RUNNING, + DAGEventType.DAG_DIAGNOSTIC_UPDATE, + DIAGNOSTIC_UPDATE_TRANSITION) + .addTransition(DAGState.RUNNING, DAGState.RUNNING, + DAGEventType.DAG_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) + .addTransition(DAGState.RUNNING, DAGState.RUNNING, + DAGEventType.DAG_SCHEDULER_UPDATE, + DAG_SCHEDULER_UPDATE_TRANSITION) + .addTransition( + DAGState.RUNNING, + DAGState.ERROR, DAGEventType.INTERNAL_ERROR, + INTERNAL_ERROR_TRANSITION) + .addTransition(DAGState.RUNNING, + EnumSet.of(DAGState.RUNNING, DAGState.TERMINATING), + DAGEventType.DAG_COMMIT_COMPLETED, + new CommitCompletedWhileRunning()) + + // Transitions from COMMITTING state. + .addTransition(DAGState.COMMITTING, + EnumSet.of(DAGState.COMMITTING, DAGState.TERMINATING, DAGState.FAILED, DAGState.SUCCEEDED), + DAGEventType.DAG_COMMIT_COMPLETED, + COMMIT_COMPLETED_TRANSITION) + .addTransition(DAGState.COMMITTING, DAGState.TERMINATING, + DAGEventType.DAG_TERMINATE, + new DAGKilledWhileCommittingTransition()) + .addTransition( + DAGState.COMMITTING, + DAGState.ERROR, + DAGEventType.INTERNAL_ERROR, + INTERNAL_ERROR_TRANSITION) + .addTransition(DAGState.COMMITTING, DAGState.COMMITTING, + DAGEventType.DAG_DIAGNOSTIC_UPDATE, + DIAGNOSTIC_UPDATE_TRANSITION) + .addTransition(DAGState.COMMITTING, DAGState.COMMITTING, + DAGEventType.DAG_SCHEDULER_UPDATE, + DAG_SCHEDULER_UPDATE_TRANSITION) + .addTransition(DAGState.COMMITTING, DAGState.TERMINATING, + DAGEventType.DAG_VERTEX_RERUNNING, + new VertexRerunWhileCommitting()) + .addTransition(DAGState.COMMITTING, DAGState.COMMITTING, + DAGEventType.DAG_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) + + // Transitions from TERMINATING state. + .addTransition + (DAGState.TERMINATING, + EnumSet.of(DAGState.TERMINATING, DAGState.KILLED, DAGState.FAILED, + DAGState.ERROR), + DAGEventType.DAG_VERTEX_COMPLETED, + new VertexCompletedTransition()) + .addTransition(DAGState.TERMINATING, DAGState.TERMINATING, + DAGEventType.DAG_DIAGNOSTIC_UPDATE, + DIAGNOSTIC_UPDATE_TRANSITION) + .addTransition(DAGState.TERMINATING, DAGState.TERMINATING, + DAGEventType.DAG_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) + .addTransition( + DAGState.TERMINATING, + DAGState.ERROR, DAGEventType.INTERNAL_ERROR, + INTERNAL_ERROR_TRANSITION) + .addTransition( + DAGState.TERMINATING, + EnumSet.of(DAGState.TERMINATING, DAGState.FAILED, DAGState.KILLED, DAGState.ERROR), + DAGEventType.DAG_COMMIT_COMPLETED, + COMMIT_COMPLETED_TRANSITION) + + // Ignore-able events + .addTransition(DAGState.TERMINATING, DAGState.TERMINATING, + EnumSet.of(DAGEventType.DAG_TERMINATE, DAGEventType.DAG_VERTEX_RERUNNING, - new VertexReRunningTransition()) - .addTransition(DAGState.RUNNING, DAGState.TERMINATING, - DAGEventType.DAG_TERMINATE, new DAGKilledTransition()) - .addTransition(DAGState.RUNNING, DAGState.RUNNING, - DAGEventType.DAG_DIAGNOSTIC_UPDATE, - DIAGNOSTIC_UPDATE_TRANSITION) - .addTransition(DAGState.RUNNING, DAGState.RUNNING, - DAGEventType.DAG_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) - .addTransition(DAGState.RUNNING, DAGState.RUNNING, + DAGEventType.DAG_SCHEDULER_UPDATE)) + + // Transitions from SUCCEEDED state + .addTransition(DAGState.SUCCEEDED, DAGState.SUCCEEDED, + DAGEventType.DAG_DIAGNOSTIC_UPDATE, + DIAGNOSTIC_UPDATE_TRANSITION) + .addTransition(DAGState.SUCCEEDED, DAGState.SUCCEEDED, + DAGEventType.DAG_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) + .addTransition( + DAGState.SUCCEEDED, + DAGState.ERROR, DAGEventType.INTERNAL_ERROR, + INTERNAL_ERROR_TRANSITION) + // Ignore-able events + .addTransition(DAGState.SUCCEEDED, DAGState.SUCCEEDED, + EnumSet.of(DAGEventType.DAG_TERMINATE, DAGEventType.DAG_SCHEDULER_UPDATE, - DAG_SCHEDULER_UPDATE_TRANSITION) - .addTransition( - DAGState.RUNNING, - DAGState.ERROR, DAGEventType.INTERNAL_ERROR, - INTERNAL_ERROR_TRANSITION) - .addTransition(DAGState.RUNNING, - EnumSet.of(DAGState.RUNNING, DAGState.TERMINATING), - DAGEventType.DAG_COMMIT_COMPLETED, - new CommitCompletedWhileRunning()) - - // Transitions from COMMITTING state. - .addTransition(DAGState.COMMITTING, - EnumSet.of(DAGState.COMMITTING, DAGState.TERMINATING, DAGState.FAILED, DAGState.SUCCEEDED), - DAGEventType.DAG_COMMIT_COMPLETED, - COMMIT_COMPLETED_TRANSITION) - .addTransition(DAGState.COMMITTING, DAGState.TERMINATING, - DAGEventType.DAG_TERMINATE, - new DAGKilledWhileCommittingTransition()) - .addTransition( - DAGState.COMMITTING, - DAGState.ERROR, - DAGEventType.INTERNAL_ERROR, - INTERNAL_ERROR_TRANSITION) - .addTransition(DAGState.COMMITTING, DAGState.COMMITTING, - DAGEventType.DAG_DIAGNOSTIC_UPDATE, - DIAGNOSTIC_UPDATE_TRANSITION) - .addTransition(DAGState.COMMITTING, DAGState.COMMITTING, + DAGEventType.DAG_VERTEX_COMPLETED)) + + // Transitions from FAILED state + .addTransition(DAGState.FAILED, DAGState.FAILED, + DAGEventType.DAG_DIAGNOSTIC_UPDATE, + DIAGNOSTIC_UPDATE_TRANSITION) + .addTransition(DAGState.FAILED, DAGState.FAILED, + DAGEventType.DAG_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) + .addTransition( + DAGState.FAILED, + DAGState.ERROR, DAGEventType.INTERNAL_ERROR, + INTERNAL_ERROR_TRANSITION) + // Ignore-able events + .addTransition(DAGState.FAILED, DAGState.FAILED, + EnumSet.of(DAGEventType.DAG_TERMINATE, + DAGEventType.DAG_START, + DAGEventType.DAG_VERTEX_RERUNNING, DAGEventType.DAG_SCHEDULER_UPDATE, - DAG_SCHEDULER_UPDATE_TRANSITION) - .addTransition(DAGState.COMMITTING, DAGState.TERMINATING, + DAGEventType.DAG_VERTEX_COMPLETED)) + + // Transitions from KILLED state + .addTransition(DAGState.KILLED, DAGState.KILLED, + DAGEventType.DAG_DIAGNOSTIC_UPDATE, + DIAGNOSTIC_UPDATE_TRANSITION) + .addTransition(DAGState.KILLED, DAGState.KILLED, + DAGEventType.DAG_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) + .addTransition( + DAGState.KILLED, + DAGState.ERROR, DAGEventType.INTERNAL_ERROR, + INTERNAL_ERROR_TRANSITION) + // Ignore-able events + .addTransition(DAGState.KILLED, DAGState.KILLED, + EnumSet.of(DAGEventType.DAG_TERMINATE, + DAGEventType.DAG_START, DAGEventType.DAG_VERTEX_RERUNNING, - new VertexRerunWhileCommitting()) - .addTransition(DAGState.COMMITTING, DAGState.COMMITTING, - DAGEventType.DAG_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) + DAGEventType.DAG_SCHEDULER_UPDATE, + DAGEventType.DAG_VERTEX_COMPLETED)) - // Transitions from TERMINATING state. - .addTransition - (DAGState.TERMINATING, - EnumSet.of(DAGState.TERMINATING, DAGState.KILLED, DAGState.FAILED, - DAGState.ERROR), + // No transitions from INTERNAL_ERROR state. Ignore all. + .addTransition( + DAGState.ERROR, + DAGState.ERROR, + EnumSet.of( + DAGEventType.DAG_TERMINATE, + DAGEventType.DAG_INIT, + DAGEventType.DAG_START, DAGEventType.DAG_VERTEX_COMPLETED, - new VertexCompletedTransition()) - .addTransition(DAGState.TERMINATING, DAGState.TERMINATING, - DAGEventType.DAG_DIAGNOSTIC_UPDATE, - DIAGNOSTIC_UPDATE_TRANSITION) - .addTransition(DAGState.TERMINATING, DAGState.TERMINATING, - DAGEventType.DAG_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) - .addTransition( - DAGState.TERMINATING, - DAGState.ERROR, DAGEventType.INTERNAL_ERROR, - INTERNAL_ERROR_TRANSITION) - .addTransition( - DAGState.TERMINATING, - EnumSet.of(DAGState.TERMINATING, DAGState.FAILED, DAGState.KILLED, DAGState.ERROR), - DAGEventType.DAG_COMMIT_COMPLETED, - COMMIT_COMPLETED_TRANSITION) - - // Ignore-able events - .addTransition(DAGState.TERMINATING, DAGState.TERMINATING, - EnumSet.of(DAGEventType.DAG_TERMINATE, - DAGEventType.DAG_VERTEX_RERUNNING, - DAGEventType.DAG_SCHEDULER_UPDATE)) - - // Transitions from SUCCEEDED state - .addTransition(DAGState.SUCCEEDED, DAGState.SUCCEEDED, - DAGEventType.DAG_DIAGNOSTIC_UPDATE, - DIAGNOSTIC_UPDATE_TRANSITION) - .addTransition(DAGState.SUCCEEDED, DAGState.SUCCEEDED, - DAGEventType.DAG_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) - .addTransition( - DAGState.SUCCEEDED, - DAGState.ERROR, DAGEventType.INTERNAL_ERROR, - INTERNAL_ERROR_TRANSITION) - // Ignore-able events - .addTransition(DAGState.SUCCEEDED, DAGState.SUCCEEDED, - EnumSet.of(DAGEventType.DAG_TERMINATE, - DAGEventType.DAG_SCHEDULER_UPDATE, - DAGEventType.DAG_VERTEX_COMPLETED)) - - // Transitions from FAILED state - .addTransition(DAGState.FAILED, DAGState.FAILED, - DAGEventType.DAG_DIAGNOSTIC_UPDATE, - DIAGNOSTIC_UPDATE_TRANSITION) - .addTransition(DAGState.FAILED, DAGState.FAILED, - DAGEventType.DAG_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) - .addTransition( - DAGState.FAILED, - DAGState.ERROR, DAGEventType.INTERNAL_ERROR, - INTERNAL_ERROR_TRANSITION) - // Ignore-able events - .addTransition(DAGState.FAILED, DAGState.FAILED, - EnumSet.of(DAGEventType.DAG_TERMINATE, - DAGEventType.DAG_START, - DAGEventType.DAG_VERTEX_RERUNNING, - DAGEventType.DAG_SCHEDULER_UPDATE, - DAGEventType.DAG_VERTEX_COMPLETED)) - - // Transitions from KILLED state - .addTransition(DAGState.KILLED, DAGState.KILLED, + DAGEventType.DAG_VERTEX_RERUNNING, + DAGEventType.DAG_SCHEDULER_UPDATE, DAGEventType.DAG_DIAGNOSTIC_UPDATE, - DIAGNOSTIC_UPDATE_TRANSITION) - .addTransition(DAGState.KILLED, DAGState.KILLED, - DAGEventType.DAG_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) - .addTransition( - DAGState.KILLED, - DAGState.ERROR, DAGEventType.INTERNAL_ERROR, - INTERNAL_ERROR_TRANSITION) - // Ignore-able events - .addTransition(DAGState.KILLED, DAGState.KILLED, - EnumSet.of(DAGEventType.DAG_TERMINATE, - DAGEventType.DAG_START, - DAGEventType.DAG_VERTEX_RERUNNING, - DAGEventType.DAG_SCHEDULER_UPDATE, - DAGEventType.DAG_VERTEX_COMPLETED)) - - // No transitions from INTERNAL_ERROR state. Ignore all. - .addTransition( - DAGState.ERROR, - DAGState.ERROR, - EnumSet.of( - DAGEventType.DAG_TERMINATE, - DAGEventType.DAG_INIT, - DAGEventType.DAG_START, - DAGEventType.DAG_VERTEX_COMPLETED, - DAGEventType.DAG_VERTEX_RERUNNING, - DAGEventType.DAG_SCHEDULER_UPDATE, - DAGEventType.DAG_DIAGNOSTIC_UPDATE, - DAGEventType.INTERNAL_ERROR, - DAGEventType.DAG_COUNTER_UPDATE)) - .addTransition(DAGState.ERROR, DAGState.ERROR, - DAGEventType.DAG_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) - // create the topology tables - .installTopology(); + DAGEventType.INTERNAL_ERROR, + DAGEventType.DAG_COUNTER_UPDATE)) + .addTransition(DAGState.ERROR, DAGState.ERROR, + DAGEventType.DAG_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) + // create the topology tables + .installTopology(); private final StateMachineTez stateMachine; @@ -505,17 +505,16 @@ public boolean isCommitted() { } } - public DAGImpl(TezDAGID dagId, - Configuration amConf, - DAGPlan jobPlan, - EventHandler eventHandler, - TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, - Credentials dagCredentials, - Clock clock, - String appUserName, - TaskHeartbeatHandler thh, - AppContext appContext) { + Configuration amConf, + DAGPlan jobPlan, + EventHandler eventHandler, + TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, + Credentials dagCredentials, + Clock clock, + String appUserName, + TaskHeartbeatHandler thh, + AppContext appContext) { this.dagId = dagId; this.jobPlan = jobPlan; this.dagConf = new Configuration(amConf); @@ -566,7 +565,7 @@ public DAGImpl(TezDAGID dagId, } else { defaultExecutionContext = null; } - + this.taskSpecificLaunchCmdOption = new TaskSpecificLaunchCmdOption(dagConf); // This "this leak" is okay because the retained pointer is in an // instance variable. @@ -594,18 +593,18 @@ private static class DagStateChangedCallback implements OnStateChangedCallback { @Override public void onStateChanged(DAGImpl dag, DAGState dagState) { - switch(dagState) { - case RUNNING: - dag.runningStatusYetToBeConsumed.set(true); - break; - case SUCCEEDED: - case FAILED: - case KILLED: - case ERROR: - dag.isFinalState.set(true); - break; - default: - break; + switch (dagState) { + case RUNNING: + dag.runningStatusYetToBeConsumed.set(true); + break; + case SUCCEEDED: + case FAILED: + case KILLED: + case ERROR: + dag.isFinalState.set(true); + break; + default: + break; } dag.dagStatusLock.lock(); try { @@ -624,7 +623,7 @@ protected StateMachine getStateMachine() { public TezDAGID getID() { return dagId; } - + @Override public Map getLocalResources() { return localResources; @@ -728,7 +727,6 @@ public TezCounters getAllCounters() { TezCounters counters = new TezCounters(); counters.incrAllCounters(dagCounters); return aggrTaskCounters(counters, vertices.values()); - } finally { readLock.unlock(); } @@ -742,7 +740,7 @@ public TezCounters getCachedCounters() { try { // FIXME a better lightweight approach for counters is needed if (fullCounters == null && cachedCounters != null - && ((cachedCountersTimestamp+10000) > System.currentTimeMillis())) { + && ((cachedCountersTimestamp + 10000) > System.currentTimeMillis())) { LOG.info("Asked for counters" + ", cachedCountersTimestamp=" + cachedCountersTimestamp + ", currentTime=" + System.currentTimeMillis()); @@ -760,7 +758,6 @@ public TezCounters getCachedCounters() { TezCounters counters = new TezCounters(); counters.incrAllCounters(dagCounters); return aggrTaskCounters(counters, vertices.values()); - } finally { readLock.unlock(); } @@ -881,7 +878,7 @@ public float getCompletedTaskProgress() { return 0.0f; } } - return ((float)completedTasks/totalTasks); + return ((float) completedTasks / totalTasks); } finally { this.readLock.unlock(); } @@ -918,7 +915,7 @@ public DAGStatusBuilder getDAGStatus(Set statusOptions) { int totalRejectedTaskAttemptCount = 0; readLock.lock(); try { - for(Map.Entry entry : vertexMap.entrySet()) { + for (Map.Entry entry : vertexMap.entrySet()) { ProgressBuilder progress = entry.getValue().getVertexProgress(); status.addVertexProgress(entry.getKey(), progress); totalTaskCount += progress.getTotalTaskCount(); @@ -1004,7 +1001,7 @@ private ProgressBuilder getDAGProgress() { int totalRejectedTaskAttemptCount = 0; readLock.lock(); try { - for(Map.Entry entry : vertexMap.entrySet()) { + for (Map.Entry entry : vertexMap.entrySet()) { ProgressBuilder progress = entry.getValue().getVertexProgress(); totalTaskCount += progress.getTotalTaskCount(); totalSucceededTaskCount += progress.getSucceededTaskCount(); @@ -1032,14 +1029,14 @@ private ProgressBuilder getDAGProgress() { @Override public VertexStatusBuilder getVertexStatus(String vertexName, - Set statusOptions) { + Set statusOptions) { Vertex vertex = vertexMap.get(vertexName); - if(vertex == null) { + if (vertex == null) { return null; } return vertex.getVertexStatus(statusOptions); } - + public TaskAttemptImpl getTaskAttempt(TezTaskAttemptID taId) { return (TaskAttemptImpl) getVertex(taId.getVertexID()).getTask(taId.getTaskID()) .getAttempt(taId); @@ -1161,7 +1158,7 @@ public Void call() throws Exception { appContext.getHadoopShim().clearHadoopCallerContext(); } } - + if (!commitEvents.isEmpty()) { try { LOG.info("Start writing dag commit event, " + getID()); @@ -1172,7 +1169,7 @@ public Void call() throws Exception { trySetTerminationCause(DAGTerminationCause.RECOVERY_FAILURE); return finished(DAGState.FAILED); } - for (Map.Entry entry : commitEvents.entrySet()) { + for (Map.Entry entry : commitEvents.entrySet()) { ListenableFuture commitFuture = appContext.getExecService().submit(entry.getValue()); Futures.addCallback(commitFuture, entry.getValue().getCallback(), GuavaShim.directExecutor()); commitFutures.put(entry.getKey(), commitFuture); @@ -1190,14 +1187,14 @@ public Void call() throws Exception { private void abortOutputs() { if (this.aborted.getAndSet(true)) { LOG.info("Ignoring multiple output abort"); - return ; + return; } // come here because dag failed or // dag succeeded and all or none semantics were on and a commit failed. // Some output may be aborted multiple times if it is shared output. // It should be OK for it to be aborted multiple times. for (Vertex vertex : vertices.values()) { - ((VertexImpl)vertex).abortVertex(VertexStatus.State.FAILED); + ((VertexImpl) vertex).abortVertex(VertexStatus.State.FAILED); } } @@ -1215,7 +1212,7 @@ public void handle(DAGEvent event) { writeLock.lock(); DAGState oldState = getInternalState(); try { - getStateMachine().doTransition(event.getType(), event); + getStateMachine().doTransition(event.getType(), event); } catch (InvalidStateTransitonException e) { String message = "Invalid event " + event.getType() + " on Dag " + this.dagId + " at currentState=" + oldState; @@ -1237,11 +1234,9 @@ public void handle(DAGEvent event) { //notify the eventhandler of state change if (oldState != getInternalState()) { LOG.info(dagId + " transitioned from " + oldState + " to " - + getInternalState() + " due to event " + event.getType()); + + getInternalState() + " due to event " + event.getType()); } - } - - finally { + } finally { writeLock.unlock(); } } @@ -1250,7 +1245,7 @@ public void handle(DAGEvent event) { public DAGState getInternalState() { readLock.lock(); try { - return getStateMachine().getCurrentState(); + return getStateMachine().getCurrentState(); } finally { readLock.unlock(); } @@ -1344,7 +1339,7 @@ static DAGState checkVerticesForCompletion(DAGImpl dag) { + ", numFailedVertices=" + dag.numFailedVertices + ", numKilledVertices=" + dag.numKilledVertices + ", numVertices=" + dag.numVertices - + ", commitInProgress=" + dag.commitFutures.size() + + ", commitInProgress=" + dag.commitFutures.size() + ", terminationCause=" + dag.terminationCause); // log in case of accounting error. @@ -1352,12 +1347,12 @@ static DAGState checkVerticesForCompletion(DAGImpl dag) { LOG.error("vertex completion accounting issue: numCompletedVertices > numVertices" + ", numCompletedVertices=" + dag.numCompletedVertices + ", numVertices=" + dag.numVertices - ); + ); } if (dag.numCompletedVertices == dag.numVertices) { //Only succeed if vertices complete successfully and no terminationCause is registered. - if(dag.numSuccessfulVertices == dag.numVertices && dag.isCommittable()) { + if (dag.numSuccessfulVertices == dag.numVertices && dag.isCommittable()) { if (dag.commitAllOutputsOnSuccess && !dag.committed.getAndSet(true)) { // start dag commit if there's any commit or just finish dag if no commit return dag.commitOrFinish(); @@ -1396,7 +1391,7 @@ static DAGState checkCommitsForCompletion(DAGImpl dag) { + ", numFailedVertices=" + dag.numFailedVertices + ", numKilledVertices=" + dag.numKilledVertices + ", numVertices=" + dag.numVertices - + ", commitInProgress=" + dag.commitFutures.size() + + ", commitInProgress=" + dag.commitFutures.size() + ", terminationCause=" + dag.terminationCause); // continue the commits if DAG#isCommittable return true, otherwise go to TERMINATING or finish dag. @@ -1424,7 +1419,7 @@ static DAGState checkCommitsForCompletion(DAGImpl dag) { private static DAGState finishWithTerminationCause(DAGImpl dag) { Preconditions.checkArgument(dag.getTerminationCause() != null, "TerminationCause is not set."); - String diagnosticMsg = "DAG did not succeed due to " + dag.terminationCause + String diagnosticMsg = "DAG did not succeed due to " + dag.terminationCause + ". failedVertices:" + dag.numFailedVertices + " killedVertices:" + dag.numKilledVertices; LOG.info(diagnosticMsg); @@ -1440,7 +1435,7 @@ private void updateCpuCounters() { dagCounters.findCounter(DAGCounter.AM_CPU_MILLISECONDS).setValue(totalDAGCpuTime); dagCounters.findCounter(DAGCounter.AM_GC_TIME_MILLIS).setValue(totalDAGGCTime); } - + private DAGState finished(DAGState finalState) { boolean dagError = false; try { @@ -1513,7 +1508,6 @@ public int getTotalVertices() { } finally { readLock.unlock(); } - } @Override @@ -1543,7 +1537,7 @@ public int getSuccessfulVertices() { * @return true if setting the value succeeded. */ boolean trySetTerminationCause(DAGTerminationCause trigger) { - if(terminationCause == null){ + if (terminationCause == null) { terminationCause = trigger; return true; } @@ -1559,7 +1553,6 @@ DAGTerminationCause getTerminationCause() { } } - DAGState initializeDAG() { commitAllOutputsOnSuccess = dagConf.getBoolean( TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, @@ -1590,7 +1583,7 @@ DAGState initializeDAG() { } // create the vertices` - for (int i=0; i < numVertices; ++i) { + for (int i = 0; i < numVertices; ++i) { String vertexName = getJobPlan().getVertex(i).getName(); VertexImpl v = createVertex(this, vertexName, i); addVertex(v); @@ -1602,7 +1595,7 @@ DAGState initializeDAG() { // TODO TEZ-2003 (post) TEZ-2624 Ideally, this should be per source. if (v.getTaskResource().compareTo(appContext.getClusterInfo().getMaxContainerCapability()) > 0) { String msg = "Vertex's TaskResource is beyond the cluster container capability," + - "Vertex=" + v.getLogIdentifier() +", Requested TaskResource=" + v.getTaskResource() + "Vertex=" + v.getLogIdentifier() + ", Requested TaskResource=" + v.getTaskResource() + ", Cluster MaxContainerCapability=" + appContext.getClusterInfo().getMaxContainerCapability(); LOG.error(msg); addDiagnostic(msg); @@ -1622,7 +1615,7 @@ DAGState initializeDAG() { finished(DAGState.FAILED); return DAGState.FAILED; } - Map edgePlans = DagTypeConverters.createEdgePlanMapFromDAGPlan(getJobPlan().getEdgeList()); + Map edgePlans = DagTypeConverters.createEdgePlanMapFromDAGPlan(getJobPlan().getEdgeList()); // setup the dag for (Vertex v : vertices.values()) { @@ -1719,8 +1712,8 @@ private static void assignDAGScheduler(DAGImpl dag) throws TezException { String dagSchedulerClassName = dag.dagConf.get(TezConfiguration.TEZ_AM_DAG_SCHEDULER_CLASS, TezConfiguration.TEZ_AM_DAG_SCHEDULER_CLASS_DEFAULT); LOG.info("Using DAG Scheduler: " + dagSchedulerClassName); - dag.dagScheduler = ReflectionUtils.createClazzInstance(dagSchedulerClassName, new Class[] { - DAG.class, EventHandler.class}, new Object[] {dag, dag.eventHandler}); + dag.dagScheduler = ReflectionUtils.createClazzInstance(dagSchedulerClassName, new Class[]{ + DAG.class, EventHandler.class}, new Object[]{dag, dag.eventHandler}); for (Vertex v : dag.vertices.values()) { dag.dagScheduler.addVertexConcurrencyLimit(v.getVertexId(), v.getMaxTaskConcurrency()); } @@ -1753,7 +1746,7 @@ private static void parseVertexEdges(DAGImpl dag, Map edgePlan Map outVertices = new HashMap(); - for(String inEdgeId : vertexPlan.getInEdgeIdList()){ + for (String inEdgeId : vertexPlan.getInEdgeIdList()) { EdgePlan edgePlan = edgePlans.get(inEdgeId); Vertex inVertex = dag.vertexMap.get(edgePlan.getInputVertexName()); Edge edge = dag.edges.get(inEdgeId); @@ -1762,7 +1755,7 @@ private static void parseVertexEdges(DAGImpl dag, Map edgePlan inVertices.put(inVertex, edge); } - for(String outEdgeId : vertexPlan.getOutEdgeIdList()){ + for (String outEdgeId : vertexPlan.getOutEdgeIdList()) { EdgePlan edgePlan = edgePlans.get(outEdgeId); Vertex outVertex = dag.vertexMap.get(edgePlan.getOutputVertexName()); Edge edge = dag.edges.get(outEdgeId); @@ -1777,7 +1770,7 @@ private static void parseVertexEdges(DAGImpl dag, Map edgePlan TezConfiguration.TEZ_AM_VERTEX_CLEANUP_HEIGHT_DEFAULT) > 0 && ShuffleUtils.isTezShuffleHandler(dag.dagConf); if (cleanupShuffleDataAtVertexLevel) { int deletionHeight = dag.dagConf.getInt(TezConfiguration.TEZ_AM_VERTEX_CLEANUP_HEIGHT, - TezConfiguration.TEZ_AM_VERTEX_CLEANUP_HEIGHT_DEFAULT); + TezConfiguration.TEZ_AM_VERTEX_CLEANUP_HEIGHT_DEFAULT); ((VertexImpl) vertex).initShuffleDeletionContext(deletionHeight); } } @@ -1802,30 +1795,30 @@ private static class RecoverTransition @Override public DAGState transition(DAGImpl dag, DAGEvent dagEvent) { - DAGEventRecoverEvent recoverEvent = (DAGEventRecoverEvent)dagEvent; + DAGEventRecoverEvent recoverEvent = (DAGEventRecoverEvent) dagEvent; // With desired state, represents the case that DAG is completed if (recoverEvent.hasDesiredState()) { VertexState vertexDesiredState = null; switch (recoverEvent.getDesiredState()) { - case SUCCEEDED: - vertexDesiredState = VertexState.SUCCEEDED; - break; - case FAILED: - vertexDesiredState = VertexState.FAILED; - break; - case KILLED: - vertexDesiredState = VertexState.KILLED; - break; - case ERROR: - vertexDesiredState = VertexState.ERROR; - break; - default: - String msg = "Invalid desired state of DAG" - + ", dagName=" + dag.getName() - + ", state=" + recoverEvent.getDesiredState(); - LOG.warn(msg); - dag.addDiagnostic(msg); - return dag.finished(DAGState.ERROR); + case SUCCEEDED: + vertexDesiredState = VertexState.SUCCEEDED; + break; + case FAILED: + vertexDesiredState = VertexState.FAILED; + break; + case KILLED: + vertexDesiredState = VertexState.KILLED; + break; + case ERROR: + vertexDesiredState = VertexState.ERROR; + break; + default: + String msg = "Invalid desired state of DAG" + + ", dagName=" + dag.getName() + + ", state=" + recoverEvent.getDesiredState(); + LOG.warn(msg); + dag.addDiagnostic(msg); + return dag.finished(DAGState.ERROR); } // Initialize dag synchronously to generate the vertices and recover its vertices to the desired state. dag.initializeDAG(); @@ -1879,14 +1872,11 @@ public DAGState transition(DAGImpl dag, DAGEvent event) { //dag.metrics.endPreparingJob(dag); dag.logJobHistoryInitedEvent(); return DAGState.INITED; - - } - } // end of InitTransition public static class StartTransition - implements SingleArcTransition { + implements SingleArcTransition { /** * This transition executes in the event-dispatcher thread, though it's * triggered in MRAppMaster's startJobs() method. @@ -1901,7 +1891,7 @@ public void transition(DAGImpl dag, DAGEvent event) { DAGEventStartDag startEvent = (DAGEventStartDag) event; List additionalUrlsForClasspath = startEvent.getAdditionalUrlsForClasspath(); if (additionalUrlsForClasspath != null) { - LOG.info("Added additional resources : [" + additionalUrlsForClasspath + "] to classpath"); + LOG.info("Added additional resources : [" + additionalUrlsForClasspath + "] to classpath"); RelocalizationUtils.addUrlsToClassPath(additionalUrlsForClasspath); } // TODO Metrics @@ -1915,6 +1905,7 @@ public void transition(DAGImpl dag, DAGEvent event) { // use LinkedHashMap to ensure the vertex order (TEZ-1065) LinkedHashMap vertexMap = new LinkedHashMap(); + void addVertex(Vertex v) { vertices.put(v.getVertexId(), v); vertexMap.put(v.getName(), v); @@ -1951,13 +1942,13 @@ public TezCounters constructFinalFullcounters() { * The vertex-kill messages are only sent once. */ void enactKill(DAGTerminationCause dagTerminationCause, - VertexTerminationCause vertexTerminationCause) { + VertexTerminationCause vertexTerminationCause) { - if(trySetTerminationCause(dagTerminationCause)){ + if (trySetTerminationCause(dagTerminationCause)) { for (Vertex v : vertices.values()) { eventHandler.handle( new VertexEventTermination(v.getVertexId(), vertexTerminationCause) - ); + ); } } } @@ -1983,7 +1974,6 @@ public DAGState transition(DAGImpl dag, DAGEvent dagEvent) { dag.addDiagnostics(event); return dag.finished(event.getTerminationCause().getFinishedState()); } - } private static class KillInitedJobTransition implements @@ -1998,7 +1988,6 @@ public DAGState transition(DAGImpl dag, DAGEvent dagEvent) { dag.addDiagnostics(event); return dag.finished(event.getTerminationCause().getFinishedState()); } - } private static class DAGKilledTransition @@ -2017,12 +2006,10 @@ public void transition(DAGImpl job, DAGEvent dagEvent) { // TODO Metrics //job.metrics.endRunningJob(job); } - - } private static class DAGKilledWhileCommittingTransition - implements SingleArcTransition { + implements SingleArcTransition { @Override public void transition(DAGImpl dag, DAGEvent dagEvent) { @@ -2064,15 +2051,13 @@ public DAGState transition(DAGImpl job, DAGEvent event) { job.numCompletedVertices++; if (vertexEvent.getVertexState() == VertexState.SUCCEEDED) { forceTransitionToKillWait = !(job.vertexSucceeded(vertex)); - } - else if (vertexEvent.getVertexState() == VertexState.FAILED) { + } else if (vertexEvent.getVertexState() == VertexState.FAILED) { job.enactKill( DAGTerminationCause.VERTEX_FAILURE, VertexTerminationCause.OTHER_VERTEX_FAILURE); job.cancelCommits(); job.vertexFailed(vertex); forceTransitionToKillWait = true; - } - else if (vertexEvent.getVertexState() == VertexState.KILLED) { + } else if (vertexEvent.getVertexState() == VertexState.KILLED) { job.vertexKilled(vertex); job.cancelCommits(); forceTransitionToKillWait = true; @@ -2089,15 +2074,13 @@ else if (vertexEvent.getVertexState() == VertexState.KILLED) { // if the job has not finished but a failure/kill occurred, then force the transition to KILL_WAIT. DAGState state = checkVerticesForCompletion(job); - if(state == DAGState.RUNNING && forceTransitionToKillWait){ + if (state == DAGState.RUNNING && forceTransitionToKillWait) { job.cancelCommits(); return DAGState.TERMINATING; - } - else { + } else { return state; } } - } private Collection getVertexIds(Collection vertexNames) { @@ -2109,7 +2092,7 @@ private Collection getVertexIds(Collection vertexNames) { } private static class VertexReRunningTransition implements - MultipleArcTransition { + MultipleArcTransition { @Override public DAGState transition(DAGImpl job, DAGEvent event) { @@ -2160,14 +2143,14 @@ private boolean vertexSucceeded(Vertex vertex) { for (String vertexName : groupInfo.groupMembers) { VertexRecoveryData vertexRecoveryData = recoveryData.getVertexRecoveryData(getVertex(vertexName).getVertexId()); - Preconditions.checkArgument(vertexRecoveryData != null,"Vertex Group has been committed" + Preconditions.checkArgument(vertexRecoveryData != null, "Vertex Group has been committed" + ", but no VertexRecoveryData found for its vertex " + vertexName); VertexFinishedEvent vertexFinishedEvent = vertexRecoveryData.getVertexFinishedEvent(); - Preconditions.checkArgument(vertexFinishedEvent!= null,"Vertex Group has been committed" + Preconditions.checkArgument(vertexFinishedEvent != null, "Vertex Group has been committed" + ", but no VertexFinishedEvent found in its vertex " + vertexName); Preconditions.checkArgument(vertexFinishedEvent.getState() == VertexState.SUCCEEDED, "Vertex Group has been committed, but unexpected vertex state of its vertex " - + vertexName + ", vertexstate=" + vertexFinishedEvent.getState()); + + vertexName + ", vertexstate=" + vertexFinishedEvent.getState()); } continue; } @@ -2192,7 +2175,9 @@ public Void call() throws Exception { LOG.info("Committing output: " + outputName); commitOutput(committer); return null; - }; + } + + ; }; ListenableFuture groupCommitFuture = appContext.getExecService().submit(groupCommitCallableEvent); Futures.addCallback(groupCommitFuture, groupCommitCallableEvent.getCallback(), @@ -2215,8 +2200,8 @@ private boolean vertexReRunning(Vertex vertex) { reRunningVertices.add(vertex.getVertexId()); numSuccessfulVertices--; addDiagnostic("Vertex re-running" - + ", vertexName=" + vertex.getName() - + ", vertexId=" + vertex.getVertexId()); + + ", vertexName=" + vertex.getName() + + ", vertexId=" + vertex.getVertexId()); if (!commitAllOutputsOnSuccess) { // partial output may already have been in committing or committed. fail if so @@ -2261,9 +2246,9 @@ private void vertexFailed(Vertex vertex) { private void vertexKilled(Vertex vertex) { numKilledVertices++; addDiagnostic("Vertex killed" - + ", vertexName=" + vertex.getName() - + ", vertexId=" + vertex.getVertexId() - + ", diagnostics=" + vertex.getDiagnostics()); + + ", vertexName=" + vertex.getName() + + ", vertexId=" + vertex.getVertexId() + + ", diagnostics=" + vertex.getDiagnostics()); // TODO: Metrics //job.metrics.killedTask(task); } @@ -2289,17 +2274,17 @@ public void transition(DAGImpl job, DAGEvent event) { for (DAGEventCounterUpdate.CounterIncrementalUpdate ci : jce .getCounterUpdates()) { job.dagCounters.findCounter(ci.getCounterKey()).increment( - ci.getIncrementValue()); + ci.getIncrementValue()); } } } private static class DAGSchedulerUpdateTransition implements - SingleArcTransition { + SingleArcTransition { @Override public void transition(DAGImpl dag, DAGEvent event) { DAGEventSchedulerUpdate sEvent = (DAGEventSchedulerUpdate) event; - switch(sEvent.getUpdateType()) { + switch (sEvent.getUpdateType()) { case TA_SCHEDULE: dag.dagScheduler.scheduleTask(sEvent); break; @@ -2308,32 +2293,31 @@ public void transition(DAGImpl dag, DAGEvent event) { break; default: throw new TezUncheckedException("Unknown DAGEventSchedulerUpdate:" - + sEvent.getUpdateType()); + + sEvent.getUpdateType()); } } } private static class CommitCompletedWhileRunning implements - MultipleArcTransition{ + MultipleArcTransition { @Override public DAGState transition(DAGImpl dag, DAGEvent event) { - DAGEventCommitCompleted commitCompletedEvent = (DAGEventCommitCompleted)event; + DAGEventCommitCompleted commitCompletedEvent = (DAGEventCommitCompleted) event; if (dag.commitCompleted(commitCompletedEvent)) { return DAGState.RUNNING; } else { return DAGState.TERMINATING; } } - } private static class CommitCompletedTransition implements - MultipleArcTransition{ + MultipleArcTransition { @Override public DAGState transition(DAGImpl dag, DAGEvent event) { - DAGEventCommitCompleted commitCompletedEvent = (DAGEventCommitCompleted)event; + DAGEventCommitCompleted commitCompletedEvent = (DAGEventCommitCompleted) event; dag.commitCompleted(commitCompletedEvent); return checkCommitsForCompletion(dag); } @@ -2348,7 +2332,7 @@ private boolean commitCompleted(DAGEventCommitCompleted commitCompletedEvent) { if (commitCompletedEvent.isSucceeded()) { LOG.info("Commit succeeded for output:" + commitCompletedEvent.getOutputKey()); OutputKey outputKey = commitCompletedEvent.getOutputKey(); - if (outputKey.isVertexGroupOutput){ + if (outputKey.isVertexGroupOutput) { VertexGroupInfo vertexGroup = vertexGroups.get(outputKey.getEntityName()); vertexGroup.successfulCommits++; if (vertexGroup.isCommitted()) { @@ -2379,21 +2363,20 @@ private boolean commitCompleted(DAGEventCommitCompleted commitCompletedEvent) { enactKill(DAGTerminationCause.COMMIT_FAILURE, VertexTerminationCause.OTHER_VERTEX_FAILURE); cancelCommits(); } - if (recoveryFailed){ + if (recoveryFailed) { enactKill(DAGTerminationCause.RECOVERY_FAILURE, VertexTerminationCause.OTHER_VERTEX_FAILURE); cancelCommits(); } return !commitFailed && !recoveryFailed; } - private static class VertexRerunWhileCommitting implements - SingleArcTransition { + SingleArcTransition { @Override public void transition(DAGImpl dag, DAGEvent event) { LOG.info("Vertex rerun while dag it is COMMITTING"); - DAGEventVertexReRunning rerunEvent = (DAGEventVertexReRunning)event; + DAGEventVertexReRunning rerunEvent = (DAGEventVertexReRunning) event; Vertex vertex = dag.getVertex(rerunEvent.getVertexId()); dag.reRunningVertices.add(vertex.getVertexId()); dag.numSuccessfulVertices--; @@ -2404,7 +2387,6 @@ public void transition(DAGImpl dag, DAGEvent event) { dag.cancelCommits(); dag.enactKill(DAGTerminationCause.VERTEX_RERUN_IN_COMMITTING, VertexTerminationCause.VERTEX_RERUN_IN_COMMITTING); } - } // TODO TEZ-2250 go to TERMINATING to wait for all vertices and commits completed @@ -2420,7 +2402,7 @@ public void transition(DAGImpl dag, DAGEvent event) { } LOG.info(dag.getID() + " terminating due to internal error. " - + (diagnostics == null? "" : " Error=" + diagnostics)); + + (diagnostics == null ? "" : " Error=" + diagnostics)); // terminate all vertices dag.enactKill(DAGTerminationCause.INTERNAL_ERROR, VertexTerminationCause.INTERNAL_ERROR); dag.setFinishTime(); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java index 78860868dd..7366031c17 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrder.java @@ -1,20 +1,20 @@ /* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.impl; @@ -30,18 +30,18 @@ @SuppressWarnings("rawtypes") public class DAGSchedulerNaturalOrder extends DAGScheduler { - - private static final Logger LOG = - LoggerFactory.getLogger(DAGSchedulerNaturalOrder.class); + + private static final Logger LOG = + LoggerFactory.getLogger(DAGSchedulerNaturalOrder.class); private final DAG dag; private final EventHandler handler; - + public DAGSchedulerNaturalOrder(DAG dag, EventHandler dispatcher) { this.dag = dag; this.handler = dispatcher; } - + @Override public void scheduleTaskEx(DAGEventSchedulerUpdate event) { TaskAttempt attempt = event.getAttempt(); @@ -55,20 +55,19 @@ public void scheduleTaskEx(DAGEventSchedulerUpdate event) { LOG.debug("Scheduling " + attempt.getTaskAttemptID() + " between priorityLow: " + priorityLowLimit + " and priorityHigh: " + priorityHighLimit); } - + TaskAttemptEventSchedule attemptEvent = new TaskAttemptEventSchedule( attempt.getTaskAttemptID(), priorityLowLimit, priorityHighLimit); - + sendEvent(attemptEvent); } - + @Override public void taskCompletedEx(DAGEventSchedulerUpdate event) { } - + @SuppressWarnings("unchecked") void sendEvent(TaskAttemptEventSchedule event) { handler.handle(event); } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrderControlled.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrderControlled.java index 3167c9f3b7..8ce8b4424e 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrderControlled.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGSchedulerNaturalOrderControlled.java @@ -241,5 +241,4 @@ public void taskCompletedEx(DAGEventSchedulerUpdate event) { private void sendEvent(TaskAttemptEventSchedule event) { handler.handle(event); } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/Edge.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/Edge.java index 08e1c19cee..78da15c6c5 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/Edge.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/Edge.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.impl; @@ -119,7 +119,7 @@ public String getSourceVertexName() { public String getDestinationVertexName() { return destinationVertex.getName(); } - + @Override public int getSourceVertexNumTasks() { return sourceVertex.getTotalTasks(); @@ -218,9 +218,9 @@ public void initialize() throws AMUserCodeException { } synchronized (this) { destinationMetaInfo = new EventMetaData(EventProducerConsumerType.INPUT, - destinationVertex.getName(), - sourceVertex.getName(), - null); + destinationVertex.getName(), + sourceVertex.getName(), + null); } } @@ -243,7 +243,7 @@ public void setEdgeProperty(EdgeProperty newEdgeProperty) throws AMUserCodeExcep sourceVertex)); } } - + // Test only method for creating specific scenarios @VisibleForTesting void setCustomEdgeManager(EdgeManagerPluginDescriptor descriptor) @@ -256,7 +256,7 @@ void setCustomEdgeManager(EdgeManagerPluginDescriptor descriptor) edgeProperty.getEdgeDestination()); setEdgeProperty(modifiedEdgeProperty); } - + public void routingToBegin() throws AMUserCodeException { int numDestTasks = edgeManagerContext.getDestinationVertexNumTasks(); synchronized (this) { @@ -264,9 +264,9 @@ public void routingToBegin() throws AMUserCodeException { routingNeeded = false; } else if (numDestTasks < 0) { throw new TezUncheckedException( - "Internal error. Not expected to route events to a destination until parallelism is determined" + - " sourceVertex=" + sourceVertex.getLogIdentifier() + - " edgeManager=" + edgeManager.getClass().getName()); + "Internal error. Not expected to route events to a destination until parallelism is determined" + + " sourceVertex=" + sourceVertex.getLogIdentifier() + + " edgeManager=" + edgeManager.getClass().getName()); } if (edgeManager instanceof EdgeManagerPluginOnDemand) { onDemandRouting = true; @@ -278,14 +278,14 @@ public void routingToBegin() throws AMUserCodeException { ((EdgeManagerPluginOnDemand) edgeManager).prepareForRouting(); } catch (Exception e) { throw new AMUserCodeException(Source.EdgeManager, - "Fail to prepareForRouting " + getEdgeInfo(), e); + "Fail to prepareForRouting " + getEdgeInfo(), e); } } - - LOG.info("Routing to begin for edge: " + getEdgeInfo() + ". EdgeProperty: " + edgeProperty + + + LOG.info("Routing to begin for edge: " + getEdgeInfo() + ". EdgeProperty: " + edgeProperty + " onDemandRouting: " + hasOnDemandRouting()); } - + public synchronized boolean hasOnDemandRouting() { return onDemandRouting; } @@ -293,7 +293,7 @@ public synchronized boolean hasOnDemandRouting() { public synchronized EdgeProperty getEdgeProperty() { return this.edgeProperty; } - + public EdgeManagerPlugin getEdgeManager() { return this.edgeManager; } @@ -316,141 +316,140 @@ public void setDestinationVertex(Vertex destinationVertex) { } public InputSpec getDestinationSpec(int destinationTaskIndex) throws AMUserCodeException { - Preconditions.checkState(edgeManager != null, + Preconditions.checkState(edgeManager != null, "Edge Manager must be initialized by this time"); try { int physicalInputCount = edgeManager.getNumDestinationTaskPhysicalInputs(destinationTaskIndex); Preconditions.checkArgument(physicalInputCount >= 0, "PhysicalInputCount should not be negative, " - + "physicalInputCount=" + physicalInputCount); + + "physicalInputCount=" + physicalInputCount); return new InputSpec(sourceVertex.getName(), edgeProperty.getEdgeDestination(), physicalInputCount); } catch (Exception e) { throw new AMUserCodeException(Source.EdgeManager, "Fail to getDestinationSpec, destinationTaskIndex=" - + destinationTaskIndex +", " + getEdgeInfo(), e); + + destinationTaskIndex + ", " + getEdgeInfo(), e); } } public OutputSpec getSourceSpec(int sourceTaskIndex) throws AMUserCodeException { - Preconditions.checkState(edgeManager != null, + Preconditions.checkState(edgeManager != null, "Edge Manager must be initialized by this time"); try { int physicalOutputCount = edgeManager.getNumSourceTaskPhysicalOutputs( sourceTaskIndex); Preconditions.checkArgument(physicalOutputCount >= 0, "PhysicalOutputCount should not be negative," - + "physicalOutputCount=" + physicalOutputCount); + + "physicalOutputCount=" + physicalOutputCount); return new OutputSpec(destinationVertex.getName(), edgeProperty.getEdgeSource(), physicalOutputCount); } catch (Exception e) { throw new AMUserCodeException(Source.EdgeManager, "Fail to getSourceSpec, sourceTaskIndex=" - + sourceTaskIndex + ", " + getEdgeInfo(), e); + + sourceTaskIndex + ", " + getEdgeInfo(), e); } } - + public void startEventBuffering() { bufferEvents.set(true); } - + public void stopEventBuffering() throws AMUserCodeException { // assume only 1 entity will start and stop event buffering bufferEvents.set(false); - for(TezEvent event : destinationEventBuffer) { + for (TezEvent event : destinationEventBuffer) { sendTezEventToDestinationTasks(event); } destinationEventBuffer.clear(); - for(TezEvent event : sourceEventBuffer) { + for (TezEvent event : sourceEventBuffer) { sendTezEventToSourceTasks(event); } sourceEventBuffer.clear(); } - + public void sendTezEventToSourceTasks(TezEvent tezEvent) throws AMUserCodeException { - Preconditions.checkState(edgeManager != null, + Preconditions.checkState(edgeManager != null, "Edge Manager must be initialized by this time"); if (!bufferEvents.get()) { switch (tezEvent.getEventType()) { - case INPUT_READ_ERROR_EVENT: - InputReadErrorEvent inputReadErrorEvent = (InputReadErrorEvent) tezEvent.getEvent(); - TezTaskAttemptID destAttemptId = tezEvent.getSourceInfo() - .getTaskAttemptID(); - int destTaskIndex = destAttemptId.getTaskID().getId(); - int srcTaskIndex; - int numConsumers; - try { - if (onDemandRouting) { - srcTaskIndex = ((EdgeManagerPluginOnDemand) edgeManager).routeInputErrorEventToSource( - destTaskIndex, inputReadErrorEvent.getIndex()); - } else { - srcTaskIndex = edgeManager.routeInputErrorEventToSource(inputReadErrorEvent, - destTaskIndex, inputReadErrorEvent.getIndex()); + case INPUT_READ_ERROR_EVENT: + InputReadErrorEvent inputReadErrorEvent = (InputReadErrorEvent) tezEvent.getEvent(); + TezTaskAttemptID destAttemptId = tezEvent.getSourceInfo() + .getTaskAttemptID(); + int destTaskIndex = destAttemptId.getTaskID().getId(); + int srcTaskIndex; + int numConsumers; + try { + if (onDemandRouting) { + srcTaskIndex = ((EdgeManagerPluginOnDemand) edgeManager).routeInputErrorEventToSource( + destTaskIndex, inputReadErrorEvent.getIndex()); + } else { + srcTaskIndex = edgeManager.routeInputErrorEventToSource(inputReadErrorEvent, + destTaskIndex, inputReadErrorEvent.getIndex()); + } + Preconditions.checkArgument(srcTaskIndex >= 0, + "SourceTaskIndex should not be negative," + + "srcTaskIndex=" + srcTaskIndex); + numConsumers = edgeManager.getNumDestinationConsumerTasks( + srcTaskIndex); + Preconditions.checkArgument(numConsumers > 0, + "ConsumerTaskNum must be positive," + + "numConsumers=" + numConsumers); + } catch (Exception e) { + throw new AMUserCodeException(Source.EdgeManager, + "Fail to sendTezEventToSourceTasks, " + + "TezEvent:" + tezEvent.getEvent() + + "sourceInfo:" + tezEvent.getSourceInfo() + + "destinationInfo:" + tezEvent.getDestinationInfo() + + ", " + getEdgeInfo(), e); } - Preconditions.checkArgument(srcTaskIndex >= 0, - "SourceTaskIndex should not be negative," - + "srcTaskIndex=" + srcTaskIndex); - numConsumers = edgeManager.getNumDestinationConsumerTasks( - srcTaskIndex); - Preconditions.checkArgument(numConsumers > 0, - "ConsumerTaskNum must be positive," - + "numConsumers=" + numConsumers); - } catch (Exception e) { - throw new AMUserCodeException(Source.EdgeManager, - "Fail to sendTezEventToSourceTasks, " - + "TezEvent:" + tezEvent.getEvent() - + "sourceInfo:" + tezEvent.getSourceInfo() - + "destinationInfo:" + tezEvent.getDestinationInfo() - + ", " + getEdgeInfo(), e); - } - Task srcTask = sourceVertex.getTask(srcTaskIndex); - if (srcTask == null) { - throw new TezUncheckedException("Unexpected null task." + - " sourceVertex=" + sourceVertex.getLogIdentifier() + - " srcIndex = " + srcTaskIndex + - " destAttemptId=" + destAttemptId + - " destIndex=" + destTaskIndex + - " edgeManager=" + edgeManager.getClass().getName()); - } - TezTaskID srcTaskId = srcTask.getTaskID(); - int srcTaskAttemptIndex = inputReadErrorEvent.getVersion(); - TezTaskAttemptID srcTaskAttemptId = TezTaskAttemptID.getInstance(srcTaskId, - srcTaskAttemptIndex); - sendEvent(new TaskAttemptEventOutputFailed(srcTaskAttemptId, tezEvent, numConsumers)); - break; - default: - throw new TezUncheckedException("Unhandled tez event type: " - + tezEvent.getEventType()); + Task srcTask = sourceVertex.getTask(srcTaskIndex); + if (srcTask == null) { + throw new TezUncheckedException("Unexpected null task." + + " sourceVertex=" + sourceVertex.getLogIdentifier() + + " srcIndex = " + srcTaskIndex + + " destAttemptId=" + destAttemptId + + " destIndex=" + destTaskIndex + + " edgeManager=" + edgeManager.getClass().getName()); + } + TezTaskID srcTaskId = srcTask.getTaskID(); + int srcTaskAttemptIndex = inputReadErrorEvent.getVersion(); + TezTaskAttemptID srcTaskAttemptId = TezTaskAttemptID.getInstance(srcTaskId, + srcTaskAttemptIndex); + sendEvent(new TaskAttemptEventOutputFailed(srcTaskAttemptId, tezEvent, numConsumers)); + break; + default: + throw new TezUncheckedException("Unhandled tez event type: " + + tezEvent.getEventType()); } } else { sourceEventBuffer.add(tezEvent); } } - private void handleCompositeDataMovementEvent(TezEvent tezEvent) throws AMUserCodeException { CompositeDataMovementEvent compEvent = (CompositeDataMovementEvent) tezEvent.getEvent(); EventMetaData srcInfo = tezEvent.getSourceInfo(); - + for (DataMovementEvent dmEvent : compEvent.getEvents()) { TezEvent newEvent = new TezEvent(dmEvent, srcInfo, tezEvent.getEventReceivedTime()); sendTezEventToDestinationTasks(newEvent); } } - + void sendDmEventOrIfEventToTasks(TezEvent tezEvent, int srcTaskIndex, - boolean isDataMovementEvent, - Map> taskAndInputIndices) { - Preconditions.checkState(edgeManager != null, + boolean isDataMovementEvent, + Map> taskAndInputIndices) { + Preconditions.checkState(edgeManager != null, "Edge Manager must be initialized by this time"); Event event = tezEvent.getEvent(); // cache of event object per input index - Map inputIndicesWithEvents = Maps.newHashMap(); + Map inputIndicesWithEvents = Maps.newHashMap(); for (Map.Entry> entry : taskAndInputIndices.entrySet()) { int destTaskIndex = entry.getKey(); List inputIndices = entry.getValue(); - for(int i=0; i listToAdd, int listMaxSize, - PendingEventRouteMetadata pendingRoutes) - throws AMUserCodeException { + int srcTaskIndex, List listToAdd, int listMaxSize, + PendingEventRouteMetadata pendingRoutes) + throws AMUserCodeException { if (!routingNeeded) { if (LOG.isDebugEnabled()) { LOG.debug("Not routing events since destination vertex has 0 tasks" + @@ -588,11 +587,10 @@ public boolean maybeAddTezEventForDestinationTask(TezEvent tezEvent, TezTaskAtte EdgeManagerPluginOnDemand edgeManagerOnDemand = (EdgeManagerPluginOnDemand) edgeManager; int taskIndex = attemptID.getTaskID().getId(); switch (tezEvent.getEventType()) { - case COMPOSITE_DATA_MOVEMENT_EVENT: - { - CompositeDataMovementEvent compEvent = (CompositeDataMovementEvent) tezEvent.getEvent(); + case COMPOSITE_DATA_MOVEMENT_EVENT: { + CompositeDataMovementEvent compEvent = (CompositeDataMovementEvent) tezEvent.getEvent(); CompositeEventRouteMetadata routeMeta = edgeManagerOnDemand - .routeCompositeDataMovementEventToDestination(srcTaskIndex, taskIndex); + .routeCompositeDataMovementEventToDestination(srcTaskIndex, taskIndex); if (routeMeta != null) { CompositeRoutedDataMovementEvent edme = compEvent.expandRouted(routeMeta); @@ -602,8 +600,7 @@ public boolean maybeAddTezEventForDestinationTask(TezEvent tezEvent, TezTaskAtte } } break; - case INPUT_FAILED_EVENT: - { + case INPUT_FAILED_EVENT: { InputFailedEvent ifEvent = (InputFailedEvent) tezEvent.getEvent(); EventRouteMetadata routeMeta; int numEventsDone; @@ -635,8 +632,7 @@ public boolean maybeAddTezEventForDestinationTask(TezEvent tezEvent, TezTaskAtte } } break; - case DATA_MOVEMENT_EVENT: - { + case DATA_MOVEMENT_EVENT: { DataMovementEvent dmEvent = (DataMovementEvent) tezEvent.getEvent(); EventRouteMetadata routeMeta; int numEventsDone; @@ -668,15 +664,15 @@ public boolean maybeAddTezEventForDestinationTask(TezEvent tezEvent, TezTaskAtte } } break; - default: - throw new TezUncheckedException("Unhandled tez event type: " - + tezEvent.getEventType()); + default: + throw new TezUncheckedException("Unhandled tez event type: " + + tezEvent.getEventType()); } - } catch (Exception e){ + } catch (Exception e) { throw new AMUserCodeException(Source.EdgeManager, "Fail to maybeAddTezEventForDestinationTask, event:" + tezEvent.getEvent() - + ", sourceInfo:" + tezEvent.getSourceInfo() + ", destinationInfo:" - + tezEvent.getDestinationInfo() + ", " + getEdgeInfo(), e); + + ", sourceInfo:" + tezEvent.getSourceInfo() + ", destinationInfo:" + + tezEvent.getDestinationInfo() + ", " + getEdgeInfo(), e); } } return true; @@ -685,8 +681,8 @@ public boolean maybeAddTezEventForDestinationTask(TezEvent tezEvent, TezTaskAtte private void sendEventToTask(Task task, TezEvent tezEvent) { task.registerTezEvent(tezEvent); } - - @SuppressWarnings({ "unchecked", "rawtypes" }) + + @SuppressWarnings({"unchecked", "rawtypes"}) private void sendEvent(org.apache.hadoop.yarn.event.Event event) { eventHandler.handle(event); } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ImmediateStartVertexManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ImmediateStartVertexManager.java index fc77e9a04d..518e952cc0 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ImmediateStartVertexManager.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ImmediateStartVertexManager.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.impl; @@ -89,7 +89,7 @@ private void scheduleTasks() { if (!canScheduleTasks()) { return; } - + tasksScheduled = true; List tasksToStart = Lists.newArrayListWithCapacity(managedTasks); for (int i = 0; i < managedTasks; ++i) { @@ -118,7 +118,7 @@ private boolean canScheduleTasks() { return true; } - + @Override public void onVertexStateUpdated(VertexStateUpdate stateUpdate) { Preconditions.checkArgument(stateUpdate.getVertexState() == VertexState.CONFIGURED, @@ -148,7 +148,6 @@ public void onVertexManagerEventReceived(VertexManagerEvent vmEvent) { @Override public void onRootVertexInitialized(String inputName, - InputDescriptor inputDescriptor, List events) { + InputDescriptor inputDescriptor, List events) { } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OneToOneEdgeManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OneToOneEdgeManager.java index 8a6008a496..b9a56be061 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OneToOneEdgeManager.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OneToOneEdgeManager.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.impl; @@ -48,32 +48,32 @@ public void initialize() { public int getNumDestinationTaskPhysicalInputs(int destinationTaskIndex) { return 1; } - + @Override public int getNumSourceTaskPhysicalOutputs(int sourceTaskIndex) { return 1; } - + @Override public void routeDataMovementEventToDestination(DataMovementEvent event, - int sourceTaskIndex, int sourceOutputIndex, - Map> destinationTaskAndInputIndices) { + int sourceTaskIndex, int sourceOutputIndex, + Map> destinationTaskAndInputIndices) { checkState(); destinationTaskAndInputIndices.put(sourceTaskIndex, destinationInputIndices); } - + @Override public void routeInputSourceTaskFailedEventToDestination(int sourceTaskIndex, - Map> destinationTaskAndInputIndices) { + Map> destinationTaskAndInputIndices) { destinationTaskAndInputIndices.put(sourceTaskIndex, destinationInputIndices); } @Override public int routeInputErrorEventToSource(InputReadErrorEvent event, - int destinationTaskIndex, int destinationFailedInputIndex) { + int destinationTaskIndex, int destinationFailedInputIndex) { return destinationTaskIndex; } - + @Override public int getNumDestinationConsumerTasks(int sourceTaskIndex) { return 1; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OneToOneEdgeManagerOnDemand.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OneToOneEdgeManagerOnDemand.java index 464d87df59..32480a7443 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OneToOneEdgeManagerOnDemand.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OneToOneEdgeManagerOnDemand.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.impl; @@ -36,8 +36,8 @@ public class OneToOneEdgeManagerOnDemand extends EdgeManagerPluginOnDemand { final List destinationInputIndices = Collections.singletonList(0); final AtomicBoolean stateChecked = new AtomicBoolean(false); - - final EventRouteMetadata commonRouteMeta = + + final EventRouteMetadata commonRouteMeta = EventRouteMetadata.create(1, new int[]{0}, new int[]{0}); final CompositeEventRouteMetadata compositeCommonRouteMeta = @@ -56,25 +56,25 @@ public void initialize() { public int getNumDestinationTaskPhysicalInputs(int destinationTaskIndex) { return 1; } - + @Override public int getNumSourceTaskPhysicalOutputs(int sourceTaskIndex) { return 1; } - + @Override public void routeDataMovementEventToDestination(DataMovementEvent event, - int sourceTaskIndex, int sourceOutputIndex, - Map> destinationTaskAndInputIndices) { + int sourceTaskIndex, int sourceOutputIndex, + Map> destinationTaskAndInputIndices) { checkState(); destinationTaskAndInputIndices.put(sourceTaskIndex, destinationInputIndices); } - + @Override public void prepareForRouting() throws Exception { checkState(); } - + @Override public EventRouteMetadata routeDataMovementEventToDestination( int sourceTaskIndex, int sourceOutputIndex, int destinationTaskIndex) @@ -84,7 +84,7 @@ public EventRouteMetadata routeDataMovementEventToDestination( } return null; } - + @Override public @Nullable CompositeEventRouteMetadata routeCompositeDataMovementEventToDestination( int sourceTaskIndex, int destinationTaskIndex) @@ -103,16 +103,16 @@ public EventRouteMetadata routeInputSourceTaskFailedEventToDestination( @Override public void routeInputSourceTaskFailedEventToDestination(int sourceTaskIndex, - Map> destinationTaskAndInputIndices) { + Map> destinationTaskAndInputIndices) { destinationTaskAndInputIndices.put(sourceTaskIndex, destinationInputIndices); } @Override public int routeInputErrorEventToSource(InputReadErrorEvent event, - int destinationTaskIndex, int destinationFailedInputIndex) { + int destinationTaskIndex, int destinationFailedInputIndex) { return destinationTaskIndex; } - + @Override public int routeInputErrorEventToSource(int destinationTaskIndex, int destinationFailedInputIndex) { return destinationTaskIndex; @@ -122,7 +122,7 @@ public int routeInputErrorEventToSource(int destinationTaskIndex, int destinatio public int getNumDestinationConsumerTasks(int sourceTaskIndex) { return 1; } - + private void checkState() { if (stateChecked.get()) { return; @@ -135,5 +135,4 @@ private void checkState() { + getContext().getSourceVertexName() + " tasks: " + getContext().getSourceVertexNumTasks()); stateChecked.set(true); } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OutputCommitterContextImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OutputCommitterContextImpl.java index dc89514950..325dfc78f2 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OutputCommitterContextImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/OutputCommitterContextImpl.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -37,11 +37,11 @@ public class OutputCommitterContextImpl implements OutputCommitterContext { private final RootInputLeafOutput output; public OutputCommitterContextImpl(ApplicationId applicationId, - int dagAttemptNumber, - String dagName, - String vertexName, - RootInputLeafOutput output, - int vertexIdx) { + int dagAttemptNumber, + String dagName, + String vertexName, + RootInputLeafOutput output, + int vertexIdx) { Objects.requireNonNull(applicationId, "applicationId is null"); Objects.requireNonNull(dagName, "dagName is null"); Objects.requireNonNull(vertexName, "vertexName is null"); @@ -83,7 +83,7 @@ public String getOutputName() { public UserPayload getOutputUserPayload() { return output.getIODescriptor().getUserPayload(); } - + @Override public UserPayload getUserPayload() { return output.getControllerDescriptor().getUserPayload(); @@ -93,5 +93,4 @@ public UserPayload getUserPayload() { public int getVertexIndex() { return vertexIdx; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/RootInputVertexManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/RootInputVertexManager.java index afe2606c1b..7d4db524d0 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/RootInputVertexManager.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/RootInputVertexManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -56,7 +56,7 @@ public class RootInputVertexManager extends VertexManagerPlugin { - private static final Logger LOG = + private static final Logger LOG = LoggerFactory.getLogger(RootInputVertexManager.class); /** @@ -87,7 +87,7 @@ public class RootInputVertexManager extends VertexManagerPlugin { "tez.root-input-vertex-manager.max-src-fraction"; public static final float TEZ_ROOT_INPUT_VERTEX_MANAGER_MAX_SRC_FRACTION_DEFAULT = 0.75f; - + private String configuredInputName; int totalNumSourceTasks = 0; @@ -117,6 +117,7 @@ public PendingTaskInfo(int index) { public String toString() { return "[index=" + index + "]"; } + public int getIndex() { return index; } @@ -129,7 +130,7 @@ static class SourceVertexInfo { int numTasks; SourceVertexInfo(final EdgeProperty edgeProperty, - int totalTasksToSchedule) { + int totalTasksToSchedule) { this.edgeProperty = edgeProperty; this.finishedTaskSet = new BitSet(); } @@ -144,7 +145,7 @@ int getNumCompletedTasks() { } SourceVertexInfo createSourceVertexInfo(EdgeProperty edgeProperty, - int numTasks) { + int numTasks) { return new SourceVertexInfo(edgeProperty, numTasks); } @@ -185,23 +186,23 @@ public void onVertexStarted(List completions) { @Override public void onVertexStateUpdated(VertexStateUpdate stateUpdate) { Preconditions.checkArgument(stateUpdate.getVertexState() == - VertexState.CONFIGURED, + VertexState.CONFIGURED, "Received incorrect state notification : " - + stateUpdate.getVertexState() + " for vertex: " - + stateUpdate.getVertexName() + " in vertex: " - + getContext().getVertexName()); + + stateUpdate.getVertexState() + " for vertex: " + + stateUpdate.getVertexName() + " in vertex: " + + getContext().getVertexName()); SourceVertexInfo vInfo = srcVertexInfo.get(stateUpdate.getVertexName()); - if(vInfo != null) { + if (vInfo != null) { Preconditions.checkState(vInfo.vertexIsConfigured == false); vInfo.vertexIsConfigured = true; vInfo.numTasks = getContext().getVertexNumTasks( stateUpdate.getVertexName()); totalNumSourceTasks += vInfo.numTasks; LOG.info("Received configured notification : {}" + " for vertex: {} in" + - " vertex: {}" + " numjourceTasks: {}", - stateUpdate.getVertexState(), stateUpdate.getVertexName(), - getContext().getVertexName(), totalNumSourceTasks); + " vertex: {}" + " numjourceTasks: {}", + stateUpdate.getVertexState(), stateUpdate.getVertexName(), + getContext().getVertexName(), totalNumSourceTasks); processPendingTasks(); } } @@ -215,7 +216,7 @@ public void onSourceTaskCompleted(TaskAttemptIdentifier attempt) { if (srcInfo.vertexIsConfigured) { Preconditions.checkState(srcTaskId < srcInfo.numTasks, "Received completion for srcTaskId " + srcTaskId + " but Vertex: " - + srcVertexName + " has only " + srcInfo.numTasks + " tasks"); + + srcVertexName + " has only " + srcInfo.numTasks + " tasks"); } // handle duplicate events and count task completions from // all source vertices @@ -241,20 +242,20 @@ public void initialize() { conf = TezUtils.createConfFromUserPayload(userPayload); } catch (IOException e) { throw new RuntimeException("Could not initialize RootInputVertexManager" - + " from provided user payload", e); + + " from provided user payload", e); } slowStartEnabled = conf.getBoolean( - TEZ_ROOT_INPUT_VERTEX_MANAGER_ENABLE_SLOW_START, - TEZ_ROOT_INPUT_VERTEX_MANAGER_ENABLE_SLOW_START_DEFAULT); + TEZ_ROOT_INPUT_VERTEX_MANAGER_ENABLE_SLOW_START, + TEZ_ROOT_INPUT_VERTEX_MANAGER_ENABLE_SLOW_START_DEFAULT); - if(slowStartEnabled) { + if (slowStartEnabled) { slowStartMinFraction = conf.getFloat( - TEZ_ROOT_INPUT_VERTEX_MANAGER_MIN_SRC_FRACTION, - TEZ_ROOT_INPUT_VERTEX_MANAGER_MIN_SRC_FRACTION_DEFAULT); + TEZ_ROOT_INPUT_VERTEX_MANAGER_MIN_SRC_FRACTION, + TEZ_ROOT_INPUT_VERTEX_MANAGER_MIN_SRC_FRACTION_DEFAULT); slowStartMaxFraction = conf.getFloat( - TEZ_ROOT_INPUT_VERTEX_MANAGER_MAX_SRC_FRACTION, - Math.max(slowStartMinFraction, - TEZ_ROOT_INPUT_VERTEX_MANAGER_MAX_SRC_FRACTION_DEFAULT)); + TEZ_ROOT_INPUT_VERTEX_MANAGER_MAX_SRC_FRACTION, + Math.max(slowStartMinFraction, + TEZ_ROOT_INPUT_VERTEX_MANAGER_MAX_SRC_FRACTION_DEFAULT)); } else { slowStartMinFraction = 0; slowStartMaxFraction = 0; @@ -263,13 +264,12 @@ public void initialize() { || slowStartMaxFraction < slowStartMinFraction) { throw new IllegalArgumentException( "Invalid values for slowStartMinFraction" - + "/slowStartMaxFraction. Min " - + "cannot be < 0, max cannot be > 1, and max cannot be < min." - + ", configuredMin=" + slowStartMinFraction - + ", configuredMax=" + slowStartMaxFraction); + + "/slowStartMaxFraction. Min " + + "cannot be < 0, max cannot be > 1, and max cannot be < min." + + ", configuredMin=" + slowStartMinFraction + + ", configuredMax=" + slowStartMaxFraction); } - updatePendingTasks(); } @@ -279,7 +279,7 @@ public void onVertexManagerEventReceived(VertexManagerEvent vmEvent) { @Override public void onRootVertexInitialized(String inputName, InputDescriptor inputDescriptor, - List events) { + List events) { List riEvents = Lists.newLinkedList(); boolean dataInformationEventSeen = false; for (Event event : events) { @@ -301,7 +301,7 @@ public void onRootVertexInitialized(String inputName, InputDescriptor inputDescr cEvent.getInputSpecUpdate() == null ? InputSpecUpdate .getDefaultSinglePhysicalInputSpecUpdate() : cEvent.getInputSpecUpdate()); getContext().reconfigureVertex(rootInputSpecUpdate, cEvent.getLocationHint(), - cEvent.getNumTasks()); + cEvent.getNumTasks()); } if (event instanceof InputUpdatePayloadEvent) { // No tasks should have been started yet. Checked by initial state check. @@ -318,8 +318,8 @@ public void onRootVertexInitialized(String inputName, InputDescriptor inputDescr + ", VertexName:" + getContext().getVertexName() + ", ConfiguredInput: " + configuredInputName + ", CurrentInput: " + inputName); configuredInputName = inputName; - - InputDataInformationEvent rEvent = (InputDataInformationEvent)event; + + InputDataInformationEvent rEvent = (InputDataInformationEvent) event; rEvent.setTargetIndex(rEvent.getSourceIndex()); // 1:1 routing riEvents.add(rEvent); } @@ -438,8 +438,8 @@ int getNumOfTasksToScheduleAndLog(float minFraction) { // numTasksToSchedule can be -ve if minFraction // is less than slowStartMinSrcCompletionFraction. LOG.info("Scheduling {} tasks for vertex: {} with totalTasks: {}. " + - "{} source tasks completed out of {}. " + - "MinSourceTaskCompletedFraction: {} min: {} max: {}", + "{} source tasks completed out of {}. " + + "MinSourceTaskCompletedFraction: {} min: {} max: {}", numTasksToSchedule, getContext().getVertexName(), totalTasksToSchedule, numSourceTasksCompleted, totalNumSourceTasks, minFraction, @@ -468,7 +468,7 @@ int getNumOfTasksToSchedule(float minSourceVertexCompletedTaskFraction) { slowStartMinFraction) / percentRange; } else { // min and max are equal. schedule 100% on reaching min - if(minSourceVertexCompletedTaskFraction < + if (minSourceVertexCompletedTaskFraction < slowStartMinFraction) { tasksFractionToSchedule = 0; } @@ -479,7 +479,7 @@ int getNumOfTasksToSchedule(float minSourceVertexCompletedTaskFraction) { // round up to avoid the corner case that single task cannot be scheduled // until src completed fraction reach max - return ((int)(Math.ceil(tasksFractionToSchedule * totalTasksToSchedule)) - + return ((int) (Math.ceil(tasksFractionToSchedule * totalTasksToSchedule)) - (totalTasksToSchedule - numPendingTasks)); } @@ -523,14 +523,14 @@ public RootInputVertexManagerConfigBuilder setSlowStart( } public RootInputVertexManagerConfigBuilder - setSlowStartMinSrcCompletionFraction(float minFraction) { + setSlowStartMinSrcCompletionFraction(float minFraction) { conf.setFloat(TEZ_ROOT_INPUT_VERTEX_MANAGER_MIN_SRC_FRACTION, minFraction); return this; } public RootInputVertexManagerConfigBuilder - setSlowStartMaxSrcCompletionFraction(float maxFraction) { + setSlowStartMaxSrcCompletionFraction(float maxFraction) { conf.setFloat(TEZ_ROOT_INPUT_VERTEX_MANAGER_MAX_SRC_FRACTION, maxFraction); return this; @@ -549,5 +549,4 @@ public VertexManagerPluginDescriptor build() { } } } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ScatterGatherEdgeManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ScatterGatherEdgeManager.java index c5b4e1aa80..6b7e4c0032 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ScatterGatherEdgeManager.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ScatterGatherEdgeManager.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.impl; @@ -35,8 +35,8 @@ import com.google.common.collect.Lists; public class ScatterGatherEdgeManager extends EdgeManagerPluginOnDemand { - - private AtomicReference> commonRouteMeta = + + private AtomicReference> commonRouteMeta = new AtomicReference>(); private Object commonRouteMetaLock = new Object(); private int[][] sourceIndices; @@ -55,7 +55,7 @@ public void initialize() { public int getNumDestinationTaskPhysicalInputs(int destinationTaskIndex) { return getContext().getSourceVertexNumTasks(); } - + @Override public int getNumSourceTaskPhysicalOutputs(int sourceTaskIndex) { int physicalOutputs = getContext().getDestinationVertexNumTasks(); @@ -73,7 +73,7 @@ private ArrayList getOrCreateCommonRouteMeta() { int numSourceTasks = getContext().getSourceVertexNumTasks(); ArrayList localEventMeta = Lists .newArrayListWithCapacity(numSourceTasks); - for (int i=0; i> destinationTaskAndInputIndices) { + int sourceTaskIndex, int sourceOutputIndex, Map> destinationTaskAndInputIndices) { // the i-th source output goes to the i-th destination task // the n-th source task becomes the n-th physical input on the task destinationTaskAndInputIndices.put(sourceOutputIndex, Collections.singletonList(sourceTaskIndex)); @@ -137,15 +137,15 @@ public void routeDataMovementEventToDestination(DataMovementEvent event, @Override public void routeInputSourceTaskFailedEventToDestination(int sourceTaskIndex, - Map> destinationTaskAndInputIndices) { - for (int i=0; i> destinationTaskAndInputIndices) { + for (int i = 0; i < getContext().getDestinationVertexNumTasks(); ++i) { destinationTaskAndInputIndices.put(i, Collections.singletonList(sourceTaskIndex)); } } @Override public int routeInputErrorEventToSource(InputReadErrorEvent event, - int destinationTaskIndex, int destinationFailedInputIndex) { + int destinationTaskIndex, int destinationFailedInputIndex) { return destinationFailedInputIndex; } @@ -158,5 +158,4 @@ public int routeInputErrorEventToSource(int destinationTaskIndex, int destinatio public int getNumDestinationConsumerTasks(int sourceTaskIndex) { return getContext().getDestinationVertexNumTasks(); } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ServicePluginInfo.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ServicePluginInfo.java index 2ff1ee5bdb..48f908a5dc 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ServicePluginInfo.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ServicePluginInfo.java @@ -18,7 +18,6 @@ package org.apache.tez.dag.app.dag.impl; - public class ServicePluginInfo { private String containerLauncherName; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java index c8343c834b..62483273c3 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -129,20 +129,24 @@ public class TaskAttemptImpl implements TaskAttempt, private static final Logger LOG = LoggerFactory.getLogger(TaskAttemptImpl.class); private static final String LINE_SEPARATOR = System .getProperty("line.separator"); - + public static class DataEventDependencyInfo { long timestamp; TezTaskAttemptID taId; + public DataEventDependencyInfo(long time, TezTaskAttemptID id) { this.timestamp = time; this.taId = id; } + public long getTimestamp() { return timestamp; } + public TezTaskAttemptID getTaskAttemptId() { return taId; } + public static DataEventDependencyInfoProto toProto(DataEventDependencyInfo info) { DataEventDependencyInfoProto.Builder builder = DataEventDependencyInfoProto.newBuilder(); builder.setTimestamp(info.timestamp); @@ -151,10 +155,10 @@ public static DataEventDependencyInfoProto toProto(DataEventDependencyInfo info) } return builder.build(); } - + public static DataEventDependencyInfo fromProto(DataEventDependencyInfoProto proto) { TezTaskAttemptID taId = null; - if(proto.hasTaskAttemptId()) { + if (proto.hasTaskAttemptId()) { taId = TezTaskAttemptID.fromString(proto.getTaskAttemptId()); } return new DataEventDependencyInfo(proto.getTimestamp(), taId); @@ -194,7 +198,7 @@ public static DataEventDependencyInfo fromProto(DataEventDependencyInfoProto pro protected NodeId containerNodeId; private String nodeHttpAddress; private String nodeRackName; - + private final Vertex vertex; private final Task task; private final TaskLocationHint locationHint; @@ -203,11 +207,11 @@ public static DataEventDependencyInfo fromProto(DataEventDependencyInfoProto pro @VisibleForTesting boolean appendNextDataEvent = true; ArrayList lastDataEvents = Lists.newArrayList(); - + @VisibleForTesting TaskAttemptStatus reportedStatus; private DAGCounter localityCounter; - + org.apache.tez.runtime.api.impl.TaskStatistics statistics; long lastNotifyProgressTimestamp = 0; @@ -225,7 +229,7 @@ public static DataEventDependencyInfo fromProto(DataEventDependencyInfoProto pro private final Resource taskResource; private final ContainerContext containerContext; private final boolean leafVertex; - + private TezTaskAttemptID creationCausalTA; private long creationTime; private long scheduledTime; @@ -235,7 +239,7 @@ public static DataEventDependencyInfo fromProto(DataEventDependencyInfoProto pro protected static final KilledTransitionHelper KILLED_HELPER = new KilledTransitionHelper(); - + private static SingleArcTransition TERMINATED_AFTER_SUCCESS_HELPER = new TerminatedAfterSuccessHelper(KILLED_HELPER); @@ -250,19 +254,19 @@ public static DataEventDependencyInfo fromProto(DataEventDependencyInfoProto pro // TA_KILLED handled the same as TA_KILL_REQUEST. Just a different name indicating a request / already killed. private static StateMachineFactory - - stateMachineFactory - = new StateMachineFactory - - (TaskAttemptStateInternal.NEW) + + stateMachineFactory + = new StateMachineFactory + + (TaskAttemptStateInternal.NEW) .addTransition(TaskAttemptStateInternal.NEW, EnumSet.of(TaskAttemptStateInternal.NEW, TaskAttemptStateInternal.START_WAIT, TaskAttemptStateInternal.FAILED), TaskAttemptEventType.TA_SCHEDULE, new ScheduleTaskattemptTransition()) - // NEW -> FAILED due to TA_FAILED happens in recovery - // (No TaskAttemptStartedEvent, but with TaskAttemptFinishedEvent(FAILED) + // NEW -> FAILED due to TA_FAILED happens in recovery + // (No TaskAttemptStartedEvent, but with TaskAttemptFinishedEvent(FAILED) .addTransition(TaskAttemptStateInternal.NEW, - TaskAttemptStateInternal.FAILED, + TaskAttemptStateInternal.FAILED, TaskAttemptEventType.TA_FAILED, new TerminateTransition(FAILED_HELPER)) .addTransition(TaskAttemptStateInternal.NEW, TaskAttemptStateInternal.KILLED, @@ -363,8 +367,6 @@ TaskAttemptEventType.TA_STARTED_REMOTELY, new StartedTransition()) TaskAttemptEventType.TA_TEZ_EVENT_UPDATE, new TezEventUpdaterTransition()) - - .addTransition(TaskAttemptStateInternal.RUNNING, TaskAttemptStateInternal.RUNNING, TaskAttemptEventType.TA_STATUS_UPDATE, STATUS_UPDATER) @@ -410,7 +412,7 @@ TaskAttemptEventType.TA_STARTED_REMOTELY, new StartedTransition()) TaskAttemptStateInternal.RUNNING), TaskAttemptEventType.TA_OUTPUT_FAILED, new OutputReportedFailedTransition()) - // for recovery, needs to log the TA generated events in TaskAttemptFinishedEvent + // for recovery, needs to log the TA generated events in TaskAttemptFinishedEvent .addTransition(TaskAttemptStateInternal.RUNNING, TaskAttemptStateInternal.RUNNING, TaskAttemptEventType.TA_TEZ_EVENT_UPDATE, @@ -528,15 +530,15 @@ TaskAttemptEventType.TA_STARTED_REMOTELY, new StartedTransition()) TaskAttemptEventType.TA_CONTAINER_TERMINATED, TaskAttemptEventType.TA_CONTAINER_TERMINATED_BY_SYSTEM)) - .installTopology(); + .installTopology(); @SuppressWarnings("rawtypes") public TaskAttemptImpl(TezTaskAttemptID attemptId, EventHandler eventHandler, - TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, Configuration conf, Clock clock, - TaskHeartbeatHandler taskHeartbeatHandler, AppContext appContext, - boolean isRescheduled, - Resource resource, ContainerContext containerContext, boolean leafVertex, - Task task, TaskLocationHint locationHint, TaskSpec taskSpec) { + TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, Configuration conf, Clock clock, + TaskHeartbeatHandler taskHeartbeatHandler, AppContext appContext, + boolean isRescheduled, + Resource resource, ContainerContext containerContext, boolean leafVertex, + Task task, TaskLocationHint locationHint, TaskSpec taskSpec) { this(attemptId, eventHandler, taskCommunicatorManagerInterface, conf, clock, taskHeartbeatHandler, appContext, isRescheduled, resource, containerContext, leafVertex, task, locationHint, taskSpec, null); @@ -544,12 +546,12 @@ public TaskAttemptImpl(TezTaskAttemptID attemptId, EventHandler eventHandler, @SuppressWarnings("rawtypes") public TaskAttemptImpl(TezTaskAttemptID attemptId, EventHandler eventHandler, - TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, Configuration conf, Clock clock, - TaskHeartbeatHandler taskHeartbeatHandler, AppContext appContext, - boolean isRescheduled, - Resource resource, ContainerContext containerContext, boolean leafVertex, - Task task, TaskLocationHint locationHint, TaskSpec taskSpec, - TezTaskAttemptID schedulingCausalTA) { + TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, Configuration conf, Clock clock, + TaskHeartbeatHandler taskHeartbeatHandler, AppContext appContext, + boolean isRescheduled, + Resource resource, ContainerContext containerContext, boolean leafVertex, + Task task, TaskLocationHint locationHint, TaskSpec taskSpec, + TezTaskAttemptID schedulingCausalTA) { ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(); this.readLock = rwLock.readLock(); @@ -577,7 +579,7 @@ public TaskAttemptImpl(TezTaskAttemptID attemptId, EventHandler eventHandler, this.containerContext = containerContext; this.leafVertex = leafVertex; this.hungIntervalMax = conf.getLong( - TezConfiguration.TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS, + TezConfiguration.TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS, TezConfiguration.TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS_DEFAULT); this.recoveryData = appContext.getDAGRecoveryData() == null ? @@ -634,7 +636,7 @@ public List getDiagnostics() { readLock.unlock(); } } - + @Override public TaskAttemptTerminationCause getTerminationCause() { return terminationCause; @@ -654,7 +656,7 @@ public TezCounters getCounters() { readLock.unlock(); } } - + TaskStatistics getStatistics() { return this.statistics; } @@ -689,10 +691,10 @@ public boolean isFinished() { readLock.lock(); try { return (EnumSet.of(TaskAttemptStateInternal.SUCCEEDED, - TaskAttemptStateInternal.FAILED, - TaskAttemptStateInternal.FAIL_IN_PROGRESS, - TaskAttemptStateInternal.KILLED, - TaskAttemptStateInternal.KILL_IN_PROGRESS) + TaskAttemptStateInternal.FAILED, + TaskAttemptStateInternal.FAIL_IN_PROGRESS, + TaskAttemptStateInternal.KILLED, + TaskAttemptStateInternal.KILL_IN_PROGRESS) .contains(getInternalState())); } finally { readLock.unlock(); @@ -708,7 +710,7 @@ public ContainerId getAssignedContainerID() { readLock.unlock(); } } - + @Override public Container getAssignedContainer() { readLock.lock(); @@ -774,7 +776,6 @@ public long getLaunchTime() { } } - /** @return task runtime duration in NS. */ public long getDurationNs() { readLock.lock(); @@ -801,7 +802,7 @@ public long getCreationTime() { readLock.unlock(); } } - + public TezTaskAttemptID getCreationCausalAttempt() { readLock.lock(); try { @@ -868,12 +869,12 @@ public void handle(TaskAttemptEvent event) { eventHandler.handle(new DAGEventDiagnosticsUpdate( getDAGID(), "Invalid event " + event.getType() + - " on TaskAttempt " + this.attemptId)); + " on TaskAttempt " + this.attemptId)); eventHandler.handle( new DAGEvent( getDAGID(), DAGEventType.INTERNAL_ERROR) - ); + ); } catch (RuntimeException e) { LOG.error("Uncaught exception when handling event " + event.getType() + " at current state " + oldState + " for " @@ -915,24 +916,24 @@ public TaskAttemptStateInternal getInternalState() { private static TaskAttemptState getExternalState( TaskAttemptStateInternal smState) { switch (smState) { - case NEW: - case START_WAIT: - case SUBMITTED: - return TaskAttemptState.STARTING; - case RUNNING: - return TaskAttemptState.RUNNING; - case FAILED: - case FAIL_IN_PROGRESS: - return TaskAttemptState.FAILED; - case KILLED: - case KILL_IN_PROGRESS: - return TaskAttemptState.KILLED; - case SUCCEEDED: - return TaskAttemptState.SUCCEEDED; - default: - throw new TezUncheckedException("Attempt to convert invalid " - + "stateMachineTaskAttemptState to externalTaskAttemptState: " - + smState); + case NEW: + case START_WAIT: + case SUBMITTED: + return TaskAttemptState.STARTING; + case RUNNING: + return TaskAttemptState.RUNNING; + case FAILED: + case FAIL_IN_PROGRESS: + return TaskAttemptState.FAILED; + case KILLED: + case KILL_IN_PROGRESS: + return TaskAttemptState.KILLED; + case SUCCEEDED: + return TaskAttemptState.SUCCEEDED; + default: + throw new TezUncheckedException("Attempt to convert invalid " + + "stateMachineTaskAttemptState to externalTaskAttemptState: " + + smState); } } @@ -957,7 +958,7 @@ private static DAGEventCounterUpdate createDAGCounterUpdateEventTALaunched( DAGEventCounterUpdate dagCounterEvent = new DAGEventCounterUpdate( ta.getDAGID() - ); + ); dagCounterEvent.addCounterUpdate(DAGCounter.TOTAL_LAUNCHED_TASKS, 1); return dagCounterEvent; } @@ -971,7 +972,7 @@ private static DAGEventCounterUpdate createDAGCounterUpdateEventTAFinished( jce.addCounterUpdate(DAGCounter.NUM_FAILED_TASKS, 1); } else if (taState == TaskAttemptState.KILLED) { jce.addCounterUpdate(DAGCounter.NUM_KILLED_TASKS, 1); - } else if (taState == TaskAttemptState.SUCCEEDED ) { + } else if (taState == TaskAttemptState.SUCCEEDED) { jce.addCounterUpdate(DAGCounter.NUM_SUCCEEDED_TASKS, 1); } @@ -1110,7 +1111,7 @@ private void sendTaskAttemptCleanupEvent() { // TezMRTypeConverter.fromTez(this.attemptId)); // sendEvent(new TaskCleanupEvent(this.attemptId, this.committer, taContext)); } - + private TaskLocationHint getTaskLocationHint() { return locationHint; } @@ -1133,7 +1134,7 @@ attemptId, getVertex().getName(), protected void logJobHistoryAttemptFinishedEvent(TaskAttemptStateInternal state) { Preconditions.checkArgument(recoveryData == null - || recoveryData.getTaskAttemptFinishedEvent() == null, + || recoveryData.getTaskAttemptFinishedEvent() == null, "log TaskAttemptFinishedEvent again in recovery when there's already another TaskAtttemptFinishedEvent"); if (getLaunchTime() == 0) return; @@ -1151,7 +1152,7 @@ attemptId, getVertex().getName(), getLaunchTime(), protected void logJobHistoryAttemptUnsuccesfulCompletion( TaskAttemptState state, TaskFailureType taskFailureType) { Preconditions.checkArgument(recoveryData == null - || recoveryData.getTaskAttemptFinishedEvent() == null, + || recoveryData.getTaskAttemptFinishedEvent() == null, "log TaskAttemptFinishedEvent again in recovery when there's already another TaskAtttemptFinishedEvent"); if (state == TaskAttemptState.FAILED && taskFailureType == null) { throw new IllegalStateException("FAILED state must be accompanied by a FailureType"); @@ -1185,9 +1186,9 @@ attemptId, getVertex().getName(), getLaunchTime(), private String getInProgressLogsUrl() { String inProgressLogsUrl = null; if (getVertex().getServicePluginInfo().getContainerLauncherName().equals( - TezConstants.getTezYarnServicePluginName()) + TezConstants.getTezYarnServicePluginName()) || getVertex().getServicePluginInfo().getContainerLauncherName().equals( - TezConstants.getTezUberServicePluginName())) { + TezConstants.getTezUberServicePluginName())) { if (containerId != null && nodeHttpAddress != null) { final String containerIdStr = containerId.toString(); inProgressLogsUrl = nodeHttpAddress @@ -1206,9 +1207,9 @@ private String getInProgressLogsUrl() { private String getCompletedLogsUrl() { String completedLogsUrl = null; if (getVertex().getServicePluginInfo().getContainerLauncherName().equals( - TezConstants.getTezYarnServicePluginName()) + TezConstants.getTezYarnServicePluginName()) || getVertex().getServicePluginInfo().getContainerLauncherName().equals( - TezConstants.getTezUberServicePluginName())) { + TezConstants.getTezUberServicePluginName())) { if (containerId != null && containerNodeId != null && nodeHttpAddress != null) { final String containerIdStr = containerId.toString(); if (conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, @@ -1236,7 +1237,7 @@ private String getCompletedLogsUrl() { ////////////////////////////////////////////////////////////////////////////// protected static class ScheduleTaskattemptTransition implements - MultipleArcTransition { + MultipleArcTransition { @Override public TaskAttemptStateInternal transition(TaskAttemptImpl ta, TaskAttemptEvent event) { @@ -1278,12 +1279,12 @@ public TaskAttemptStateInternal transition(TaskAttemptImpl ta, TaskAttemptEvent taFinishedEvent.getDiagnostics(), taFinishedEvent.getTaskAttemptError(), true)); break; case SUCCEEDED: - LOG.debug("TaskAttemptFinishedEvent is seen with state of SUCCEEDED, " - + "send TA_DONE to itself, attemptId={}", ta.attemptId); + LOG.debug("TaskAttemptFinishedEvent is seen with state of SUCCEEDED, " + + "send TA_DONE to itself, attemptId={}", ta.attemptId); ta.sendEvent(new TaskAttemptEvent(ta.getTaskAttemptID(), TaskAttemptEventType.TA_DONE)); break; default: - throw new TezUncheckedException("Invalid state in TaskAttemptFinishedEvent, state=" + throw new TezUncheckedException("Invalid state in TaskAttemptFinishedEvent, state=" + taFinishedEvent.getState() + ", taId=" + ta.getTaskAttemptID()); } return TaskAttemptStateInternal.NEW; @@ -1322,10 +1323,10 @@ public TaskAttemptStateInternal transition(TaskAttemptImpl ta, TaskAttemptEvent } LOG.debug("Asking for container launch with taskAttemptContext: {}", ta.taskSpec); - + // Send out a launch request to the scheduler. int priority; - if (ta.isRescheduled && ta.getVertex().getVertexConfig().getTaskRescheduleHigherPriority()) { + if (ta.isRescheduled && ta.getVertex().getVertexConfig().getTaskRescheduleHigherPriority()) { // higher priority for rescheduled attempts priority = scheduleEvent.getPriorityHighLimit(); } else { @@ -1368,8 +1369,8 @@ public void transition(TaskAttemptImpl ta, TaskAttemptEvent event) { ta.trySetTerminationCause(((TaskAttemptEventTerminationCauseEvent) event).getTerminationCause()); } else { throw new TezUncheckedException("Invalid event received in TerminateTransition" - + ", requiredClass=TaskAttemptEventTerminationCauseEvent" - + ", eventClass=" + event.getClass().getName()); + + ", requiredClass=TaskAttemptEventTerminationCauseEvent" + + ", eventClass=" + event.getClass().getName()); } if (event instanceof TaskAttemptEventContainerTerminated) { @@ -1407,7 +1408,7 @@ public void transition(TaskAttemptImpl ta, TaskAttemptEvent event) { } if (event instanceof RecoveryEvent) { - RecoveryEvent rEvent = (RecoveryEvent)event; + RecoveryEvent rEvent = (RecoveryEvent) event; if (rEvent.isFromRecovery()) { if (LOG.isDebugEnabled()) { LOG.debug("Faked TerminateEvent from recovery, taskAttemptId=" + ta.getTaskAttemptID()); @@ -1497,7 +1498,7 @@ public void transition(TaskAttemptImpl ta, TaskAttemptEvent taskAttemptEvent) { // only and moves the attempt to a final state, or an initial state. } } - + private boolean isSpeculationEnabled() { return conf.getBoolean(TezConfiguration.TEZ_AM_SPECULATION_ENABLED, TezConfiguration.TEZ_AM_SPECULATION_ENABLED_DEFAULT); @@ -1510,7 +1511,7 @@ public TerminatedBeforeRunningTransition( TerminatedTransitionHelper helper) { super(helper); } - + protected boolean sendSchedulerEvent() { return true; } @@ -1522,7 +1523,7 @@ public void transition(TaskAttemptImpl ta, TaskAttemptEvent event) { if (sendSchedulerEvent()) { ta.sendEvent(new AMSchedulerEventTAEnded(ta, ta.containerId, helper .getTaskAttemptState(), TezUtilsInternal.toTaskAttemptEndReason(ta.terminationCause), - ta instanceof DiagnosableEvent ? ((DiagnosableEvent)ta).getDiagnosticInfo() : null, + ta instanceof DiagnosableEvent ? ((DiagnosableEvent) ta).getDiagnosticInfo() : null, ta.getVertex().getTaskSchedulerIdentifier())); } } @@ -1559,7 +1560,7 @@ protected static class ContainerCompletedBeforeRunningTransition extends public ContainerCompletedBeforeRunningTransition() { super(FAILED_HELPER); } - + public ContainerCompletedBeforeRunningTransition(TerminatedTransitionHelper helper) { super(helper); } @@ -1569,14 +1570,13 @@ public void transition(TaskAttemptImpl ta, TaskAttemptEvent event) { super.transition(ta, event); ta.sendTaskAttemptCleanupEvent(); } - } protected static class StatusUpdaterTransition implements SingleArcTransition { @Override public void transition(TaskAttemptImpl ta, TaskAttemptEvent event) { - TaskAttemptEventStatusUpdate sEvent = (TaskAttemptEventStatusUpdate) event; + TaskAttemptEventStatusUpdate sEvent = (TaskAttemptEventStatusUpdate) event; TaskStatusUpdateEvent statusEvent = sEvent.getStatusEvent(); ta.reportedStatus.state = ta.getState(); ta.reportedStatus.progress = statusEvent.getProgress(); @@ -1593,20 +1593,20 @@ public void transition(TaskAttemptImpl ta, TaskAttemptEvent event) { if (ta.hungIntervalMax > 0 && ta.lastNotifyProgressTimestamp > 0 && currTime - ta.lastNotifyProgressTimestamp > ta.hungIntervalMax) { // task is hung - String diagnostics = "Attempt failed because it appears to make no progress for " + - ta.hungIntervalMax + "ms"; + String diagnostics = "Attempt failed because it appears to make no progress for " + + ta.hungIntervalMax + "ms"; LOG.info(diagnostics + " " + ta.getTaskAttemptID()); // send event that will fail this attempt ta.sendEvent( new TaskAttemptEventAttemptFailed(ta.getTaskAttemptID(), TaskAttemptEventType.TA_FAILED, TaskFailureType.NON_FATAL, - diagnostics, + diagnostics, TaskAttemptTerminationCause.NO_PROGRESS) - ); + ); } } - + if (sEvent.getReadErrorReported()) { // if there is a read error then track the next last data event ta.appendNextDataEvent = true; @@ -1626,7 +1626,7 @@ protected static class TezEventUpdaterTransition implements @Override public void transition(TaskAttemptImpl ta, TaskAttemptEvent event) { - TaskAttemptEventTezEventUpdate tezEventUpdate = (TaskAttemptEventTezEventUpdate)event; + TaskAttemptEventTezEventUpdate tezEventUpdate = (TaskAttemptEventTezEventUpdate) event; ta.taGeneratedEvents.addAll(tezEventUpdate.getTezEvents()); } } @@ -1668,10 +1668,10 @@ public void transition(TaskAttemptImpl ta, TaskAttemptEvent event) { // Unregister from the TaskHeartbeatHandler. ta.taskHeartbeatHandler.unregister(ta.attemptId); - + ta.reportedStatus.state = TaskAttemptState.SUCCEEDED; ta.reportedStatus.progress = 1.0f; - + if (ta.isSpeculationEnabled()) { ta.sendEvent(new SpeculatorEventTaskAttemptStatusUpdate(ta.attemptId, TaskAttemptState.SUCCEEDED, ta.clock.getTime())); @@ -1708,7 +1708,7 @@ protected static class ContainerCompletedWhileRunningTransition extends public ContainerCompletedWhileRunningTransition() { super(FAILED_HELPER); } - + public ContainerCompletedWhileRunningTransition(TerminatedTransitionHelper helper) { super(helper); } @@ -1729,7 +1729,6 @@ public void transition(TaskAttemptImpl ta, TaskAttemptEvent event) { TaskAttemptEventContainerTerminated tEvent = (TaskAttemptEventContainerTerminated) event; ta.addDiagnosticInfo(tEvent.getDiagnosticInfo()); } - } protected static class TerminatedAfterSuccessHelper extends @@ -1741,7 +1740,7 @@ protected boolean sendSchedulerEvent() { // there is no need to send it again return false; } - + public TerminatedAfterSuccessHelper(TerminatedTransitionHelper helper) { super(helper); } @@ -1751,7 +1750,6 @@ public void transition(TaskAttemptImpl ta, TaskAttemptEvent event) { super.transition(ta, event); ta.sendTaskAttemptCleanupEvent(); } - } protected static class TerminatedAfterSuccessTransition implements @@ -1770,19 +1768,19 @@ public TaskAttemptStateInternal transition(TaskAttemptImpl attempt, TaskAttemptE return TaskAttemptStateInternal.KILLED; } } - + protected static class OutputReportedFailedTransition implements - MultipleArcTransition { + MultipleArcTransition { @Override public TaskAttemptStateInternal transition(TaskAttemptImpl sourceAttempt, - TaskAttemptEvent event) { - TaskAttemptEventOutputFailed outputFailedEvent = + TaskAttemptEvent event) { + TaskAttemptEventOutputFailed outputFailedEvent = (TaskAttemptEventOutputFailed) event; TezEvent inputFailedEvent = outputFailedEvent.getInputFailedEvent(); TezTaskAttemptID failedDestTaId = inputFailedEvent.getSourceInfo().getTaskAttemptID(); - InputReadErrorEvent readErrorEvent = (InputReadErrorEvent)inputFailedEvent.getEvent(); + InputReadErrorEvent readErrorEvent = (InputReadErrorEvent) inputFailedEvent.getEvent(); int failedInputIndexOnDestTa = readErrorEvent.getIndex(); if (readErrorEvent.getVersion() != sourceAttempt.getTaskAttemptID().getId()) { @@ -1834,7 +1832,7 @@ public TaskAttemptStateInternal transition(TaskAttemptImpl sourceAttempt, double maxAllowedOutputFailuresFraction = sourceAttempt.getVertex() .getVertexConfig().getMaxAllowedOutputFailuresFraction(); - int readErrorTimespanSec = (int)((time - firstErrReportTime)/1000); + int readErrorTimespanSec = (int) ((time - firstErrReportTime) / 1000); boolean crossTimeDeadline = readErrorTimespanSec >= maxAllowedTimeForTaskReadErrorSec; int runningTasks = sourceAttempt.appContext.getCurrentDAG().getVertex( @@ -1900,7 +1898,7 @@ private int getNumNodes(TaskAttemptImpl sourceAttempt) { return numActiveNodes; } } - + @VisibleForTesting protected void sendInputFailedToConsumers() { Vertex vertex = getVertex(); @@ -1908,16 +1906,16 @@ protected void sendInputFailedToConsumers() { if (edges != null && !edges.isEmpty()) { List tezIfEvents = Lists.newArrayListWithCapacity(edges.size()); for (Vertex edgeVertex : edges.keySet()) { - tezIfEvents.add(new TezEvent(new InputFailedEvent(), - new EventMetaData(EventProducerConsumerType.SYSTEM, - vertex.getName(), + tezIfEvents.add(new TezEvent(new InputFailedEvent(), + new EventMetaData(EventProducerConsumerType.SYSTEM, + vertex.getName(), edgeVertex.getName(), - getTaskAttemptID()), appContext.getClock().getTime())); + getTaskAttemptID()), appContext.getClock().getTime())); } sendEvent(new VertexEventRouteEvent(vertex.getVertexId(), tezIfEvents)); } } - + private void trySetTerminationCause(TaskAttemptTerminationCause err) { // keep only the first error cause if (terminationCause == TaskAttemptTerminationCause.UNKNOWN_ERROR) { @@ -1971,7 +1969,7 @@ public TaskEvent getTaskEvent(TezTaskAttemptID taskAttemptId, @Override public TaskFailureType getFailureType(TaskAttemptEvent event) { if (event instanceof TaskAttemptEventAttemptFailed) { - return ( ((TaskAttemptEventAttemptFailed) event).getTaskFailureType()); + return (((TaskAttemptEventAttemptFailed) event).getTaskFailureType()); } else { // For alternate failure scenarios like OUTPUT_FAILED, CONTAINER_TERMINATING, NODE_FAILED, etc return TaskFailureType.NON_FATAL; @@ -2009,7 +2007,6 @@ public String toString() { return getTaskAttemptID().toString(); } - @Override public void setLastEventSent(TezEvent lastEventSent) { writeLock.lock(); @@ -2017,12 +2014,12 @@ public void setLastEventSent(TezEvent lastEventSent) { // TEZ-3066 ideally Heartbeat just happens in FAIL_IN_PROGRESS & KILL_IN_PROGRESS, // add other states here just in case. create TEZ-3068 for a more elegant solution. if (!EnumSet.of(TaskAttemptStateInternal.FAIL_IN_PROGRESS, - TaskAttemptStateInternal.KILL_IN_PROGRESS, - TaskAttemptStateInternal.FAILED, - TaskAttemptStateInternal.KILLED, - TaskAttemptStateInternal.SUCCEEDED).contains(getInternalState())) { + TaskAttemptStateInternal.KILL_IN_PROGRESS, + TaskAttemptStateInternal.FAILED, + TaskAttemptStateInternal.KILLED, + TaskAttemptStateInternal.SUCCEEDED).contains(getInternalState())) { DataEventDependencyInfo info = new DataEventDependencyInfo( - lastEventSent.getEventReceivedTime(), lastEventSent.getSourceInfo().getTaskAttemptID()); + lastEventSent.getEventReceivedTime(), lastEventSent.getSourceInfo().getTaskAttemptID()); // task attempt id may be null for input data information events if (appendNextDataEvent) { appendNextDataEvent = false; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImplHelpers.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImplHelpers.java index 9e4f2b4a6e..c5e19f493f 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImplHelpers.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImplHelpers.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -28,7 +28,7 @@ public class TaskAttemptImplHelpers { private static final Logger LOG = LoggerFactory.getLogger(TaskAttemptImplHelpers.class); - + static String[] resolveHosts(String[] src) { String[] result = new String[src.length]; for (int i = 0; i < src.length; i++) { @@ -54,8 +54,8 @@ static String resolveHost(String src) { } private static final Pattern ipPattern = // Pattern for matching ip - Pattern.compile("\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}"); - + Pattern.compile("\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}"); + static boolean isIP(String src) { return ipPattern.matcher(src).matches(); } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java index d2933c5b86..39e0da28d0 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskImpl.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.impl; @@ -111,13 +111,13 @@ /** * Implementation of Task interface. */ -@SuppressWarnings({ "rawtypes", "unchecked" }) +@SuppressWarnings({"rawtypes", "unchecked"}) public class TaskImpl implements Task, EventHandler { private static final Logger LOG = LoggerFactory.getLogger(TaskImpl.class); private static final String LINE_SEPARATOR = System - .getProperty("line.separator"); + .getProperty("line.separator"); protected final Configuration conf; protected final TaskCommunicatorManagerInterface taskCommunicatorManagerInterface; @@ -151,7 +151,7 @@ public class TaskImpl implements Task, EventHandler { new ArrayList(0); // track the status of TaskAttempt (true mean completed, false mean uncompleted) - private final Map taskAttemptStatus = new HashMap(); + private final Map taskAttemptStatus = new HashMap(); // The set of nodes with active running attempts at the time of the latest attempt for // this task was scheduled. This set is empty when scheduling original task attempt, and @@ -161,155 +161,154 @@ public class TaskImpl implements Task, EventHandler { .newSetFromMap(new ConcurrentHashMap()); private static final MultipleArcTransition - ATTEMPT_KILLED_TRANSITION = new AttemptKilledTransition(); + ATTEMPT_KILLED_TRANSITION = new AttemptKilledTransition(); private static final SingleArcTransition - KILL_TRANSITION = new KillTransition(); + KILL_TRANSITION = new KillTransition(); private static final SingleArcTransition REDUNDANT_COMPLETED_TRANSITION = new AttemptRedundantCompletedTransition(); - private static final TaskStateChangedCallback STATE_CHANGED_CALLBACK = new TaskStateChangedCallback(); - + private static final StateMachineFactory - - stateMachineFactory - = new StateMachineFactory - (TaskStateInternal.NEW) - - // define the state machine of Task - - // Transitions from NEW state - // Stay in NEW in recovery when Task is killed in the previous AM - .addTransition(TaskStateInternal.NEW, - EnumSet.of(TaskStateInternal.NEW, TaskStateInternal.SCHEDULED), - TaskEventType.T_SCHEDULE, new InitialScheduleTransition()) - .addTransition(TaskStateInternal.NEW, TaskStateInternal.KILLED, - TaskEventType.T_TERMINATE, - new KillNewTransition()) - - // Transitions from SCHEDULED state + + stateMachineFactory + = new StateMachineFactory + (TaskStateInternal.NEW) + + // define the state machine of Task + + // Transitions from NEW state + // Stay in NEW in recovery when Task is killed in the previous AM + .addTransition(TaskStateInternal.NEW, + EnumSet.of(TaskStateInternal.NEW, TaskStateInternal.SCHEDULED), + TaskEventType.T_SCHEDULE, new InitialScheduleTransition()) + .addTransition(TaskStateInternal.NEW, TaskStateInternal.KILLED, + TaskEventType.T_TERMINATE, + new KillNewTransition()) + + // Transitions from SCHEDULED state //when the first attempt is launched, the task state is set to RUNNING - .addTransition(TaskStateInternal.SCHEDULED, TaskStateInternal.RUNNING, - TaskEventType.T_ATTEMPT_LAUNCHED, new LaunchTransition()) - .addTransition(TaskStateInternal.SCHEDULED, TaskStateInternal.KILL_WAIT, - TaskEventType.T_TERMINATE, - KILL_TRANSITION) - .addTransition(TaskStateInternal.SCHEDULED, EnumSet.of(TaskStateInternal.SCHEDULED, TaskStateInternal.FAILED), - TaskEventType.T_ATTEMPT_KILLED, ATTEMPT_KILLED_TRANSITION) - .addTransition(TaskStateInternal.SCHEDULED, - EnumSet.of(TaskStateInternal.SCHEDULED, TaskStateInternal.FAILED), - TaskEventType.T_ATTEMPT_FAILED, - new AttemptFailedTransition()) - // Happens in recovery - .addTransition(TaskStateInternal.SCHEDULED, - EnumSet.of(TaskStateInternal.RUNNING, TaskStateInternal.SUCCEEDED), - TaskEventType.T_ATTEMPT_SUCCEEDED, - new AttemptSucceededTransition()) - - // When current attempt fails/killed and new attempt launched then - // TODO Task should go back to SCHEDULED state TEZ-495 - // Transitions from RUNNING state - .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING, - TaskEventType.T_ATTEMPT_LAUNCHED) //more attempts may start later - .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING, - TaskEventType.T_ADD_SPEC_ATTEMPT, new RedundantScheduleTransition()) - .addTransition(TaskStateInternal.RUNNING, - EnumSet.of(TaskStateInternal.SUCCEEDED), - TaskEventType.T_ATTEMPT_SUCCEEDED, - new AttemptSucceededTransition()) - .addTransition(TaskStateInternal.RUNNING, EnumSet.of(TaskStateInternal.RUNNING, TaskStateInternal.FAILED), - TaskEventType.T_ATTEMPT_KILLED, - ATTEMPT_KILLED_TRANSITION) - .addTransition(TaskStateInternal.RUNNING, - EnumSet.of(TaskStateInternal.RUNNING, TaskStateInternal.FAILED), - TaskEventType.T_ATTEMPT_FAILED, - new AttemptFailedTransition()) - .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.KILL_WAIT, - TaskEventType.T_TERMINATE, - KILL_TRANSITION) - .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING, - TaskEventType.T_SCHEDULE) - - // Transitions from KILL_WAIT state - .addTransition(TaskStateInternal.KILL_WAIT, - EnumSet.of(TaskStateInternal.KILL_WAIT, TaskStateInternal.KILLED), - TaskEventType.T_ATTEMPT_KILLED, - new KillWaitAttemptCompletedTransition()) - .addTransition(TaskStateInternal.KILL_WAIT, - EnumSet.of(TaskStateInternal.KILL_WAIT, TaskStateInternal.KILLED), - TaskEventType.T_ATTEMPT_FAILED, - new KillWaitAttemptCompletedTransition()) - .addTransition(TaskStateInternal.KILL_WAIT, - EnumSet.of(TaskStateInternal.KILL_WAIT, TaskStateInternal.KILLED), - TaskEventType.T_ATTEMPT_SUCCEEDED, - new KillWaitAttemptCompletedTransition()) - // Ignore-able transitions. - .addTransition( - TaskStateInternal.KILL_WAIT, - TaskStateInternal.KILL_WAIT, - EnumSet.of( - TaskEventType.T_TERMINATE, - TaskEventType.T_ATTEMPT_LAUNCHED, - TaskEventType.T_ADD_SPEC_ATTEMPT)) - - // Transitions from SUCCEEDED state - .addTransition(TaskStateInternal.SUCCEEDED, //only possible for map tasks - EnumSet.of(TaskStateInternal.SCHEDULED, TaskStateInternal.SUCCEEDED, TaskStateInternal.FAILED), - TaskEventType.T_ATTEMPT_FAILED, new TaskRetroactiveFailureTransition()) - .addTransition(TaskStateInternal.SUCCEEDED, //only possible for map tasks - EnumSet.of(TaskStateInternal.SCHEDULED, TaskStateInternal.SUCCEEDED), - TaskEventType.T_ATTEMPT_KILLED, new TaskRetroactiveKilledTransition()) - .addTransition( - TaskStateInternal.SUCCEEDED, TaskStateInternal.SUCCEEDED, - TaskEventType.T_ATTEMPT_SUCCEEDED, REDUNDANT_COMPLETED_TRANSITION) - // Ignore-able transitions. - .addTransition(TaskStateInternal.SUCCEEDED, TaskStateInternal.SUCCEEDED, - EnumSet.of( - TaskEventType.T_ADD_SPEC_ATTEMPT, - TaskEventType.T_TERMINATE, - TaskEventType.T_ATTEMPT_LAUNCHED)) - .addTransition(TaskStateInternal.SUCCEEDED, TaskStateInternal.SUCCEEDED, - TaskEventType.T_SCHEDULE) - - // Transitions from FAILED state - .addTransition(TaskStateInternal.FAILED, TaskStateInternal.FAILED, - EnumSet.of( - TaskEventType.T_ATTEMPT_FAILED, - TaskEventType.T_ATTEMPT_KILLED, - TaskEventType.T_ATTEMPT_SUCCEEDED), REDUNDANT_COMPLETED_TRANSITION) - .addTransition(TaskStateInternal.FAILED, TaskStateInternal.FAILED, - EnumSet.of( - TaskEventType.T_TERMINATE, - TaskEventType.T_SCHEDULE, - TaskEventType.T_ADD_SPEC_ATTEMPT, - TaskEventType.T_ATTEMPT_LAUNCHED)) - - // Transitions from KILLED state - // Ignorable event: T_ATTEMPT_KILLED - // Refer to TEZ-2379 - // T_ATTEMPT_KILLED can show up in KILLED state as - // a SUCCEEDED attempt can still transition to KILLED after receiving - // a KILL event. - // This could happen when there is a race where the task receives a - // kill event, it tries to kill all running attempts and a potential - // running attempt succeeds before it receives the kill event. - // The task will then receive both a SUCCEEDED and KILLED - // event from the same attempt. - // Duplicate events from a single attempt in KILL_WAIT are handled - // properly. However, the subsequent T_ATTEMPT_KILLED event might - // be received after the task reaches its terminal KILLED state. - .addTransition(TaskStateInternal.KILLED, TaskStateInternal.KILLED, - EnumSet.of( - TaskEventType.T_TERMINATE, - TaskEventType.T_SCHEDULE, - TaskEventType.T_ADD_SPEC_ATTEMPT, - TaskEventType.T_ATTEMPT_LAUNCHED, - TaskEventType.T_ATTEMPT_SUCCEEDED, - TaskEventType.T_ATTEMPT_FAILED, - TaskEventType.T_ATTEMPT_KILLED)) - - // create the topology tables - .installTopology(); + .addTransition(TaskStateInternal.SCHEDULED, TaskStateInternal.RUNNING, + TaskEventType.T_ATTEMPT_LAUNCHED, new LaunchTransition()) + .addTransition(TaskStateInternal.SCHEDULED, TaskStateInternal.KILL_WAIT, + TaskEventType.T_TERMINATE, + KILL_TRANSITION) + .addTransition(TaskStateInternal.SCHEDULED, EnumSet.of(TaskStateInternal.SCHEDULED, TaskStateInternal.FAILED), + TaskEventType.T_ATTEMPT_KILLED, ATTEMPT_KILLED_TRANSITION) + .addTransition(TaskStateInternal.SCHEDULED, + EnumSet.of(TaskStateInternal.SCHEDULED, TaskStateInternal.FAILED), + TaskEventType.T_ATTEMPT_FAILED, + new AttemptFailedTransition()) + // Happens in recovery + .addTransition(TaskStateInternal.SCHEDULED, + EnumSet.of(TaskStateInternal.RUNNING, TaskStateInternal.SUCCEEDED), + TaskEventType.T_ATTEMPT_SUCCEEDED, + new AttemptSucceededTransition()) + + // When current attempt fails/killed and new attempt launched then + // TODO Task should go back to SCHEDULED state TEZ-495 + // Transitions from RUNNING state + .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING, + TaskEventType.T_ATTEMPT_LAUNCHED) //more attempts may start later + .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING, + TaskEventType.T_ADD_SPEC_ATTEMPT, new RedundantScheduleTransition()) + .addTransition(TaskStateInternal.RUNNING, + EnumSet.of(TaskStateInternal.SUCCEEDED), + TaskEventType.T_ATTEMPT_SUCCEEDED, + new AttemptSucceededTransition()) + .addTransition(TaskStateInternal.RUNNING, EnumSet.of(TaskStateInternal.RUNNING, TaskStateInternal.FAILED), + TaskEventType.T_ATTEMPT_KILLED, + ATTEMPT_KILLED_TRANSITION) + .addTransition(TaskStateInternal.RUNNING, + EnumSet.of(TaskStateInternal.RUNNING, TaskStateInternal.FAILED), + TaskEventType.T_ATTEMPT_FAILED, + new AttemptFailedTransition()) + .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.KILL_WAIT, + TaskEventType.T_TERMINATE, + KILL_TRANSITION) + .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING, + TaskEventType.T_SCHEDULE) + + // Transitions from KILL_WAIT state + .addTransition(TaskStateInternal.KILL_WAIT, + EnumSet.of(TaskStateInternal.KILL_WAIT, TaskStateInternal.KILLED), + TaskEventType.T_ATTEMPT_KILLED, + new KillWaitAttemptCompletedTransition()) + .addTransition(TaskStateInternal.KILL_WAIT, + EnumSet.of(TaskStateInternal.KILL_WAIT, TaskStateInternal.KILLED), + TaskEventType.T_ATTEMPT_FAILED, + new KillWaitAttemptCompletedTransition()) + .addTransition(TaskStateInternal.KILL_WAIT, + EnumSet.of(TaskStateInternal.KILL_WAIT, TaskStateInternal.KILLED), + TaskEventType.T_ATTEMPT_SUCCEEDED, + new KillWaitAttemptCompletedTransition()) + // Ignore-able transitions. + .addTransition( + TaskStateInternal.KILL_WAIT, + TaskStateInternal.KILL_WAIT, + EnumSet.of( + TaskEventType.T_TERMINATE, + TaskEventType.T_ATTEMPT_LAUNCHED, + TaskEventType.T_ADD_SPEC_ATTEMPT)) + + // Transitions from SUCCEEDED state + .addTransition(TaskStateInternal.SUCCEEDED, //only possible for map tasks + EnumSet.of(TaskStateInternal.SCHEDULED, TaskStateInternal.SUCCEEDED, TaskStateInternal.FAILED), + TaskEventType.T_ATTEMPT_FAILED, new TaskRetroactiveFailureTransition()) + .addTransition(TaskStateInternal.SUCCEEDED, //only possible for map tasks + EnumSet.of(TaskStateInternal.SCHEDULED, TaskStateInternal.SUCCEEDED), + TaskEventType.T_ATTEMPT_KILLED, new TaskRetroactiveKilledTransition()) + .addTransition( + TaskStateInternal.SUCCEEDED, TaskStateInternal.SUCCEEDED, + TaskEventType.T_ATTEMPT_SUCCEEDED, REDUNDANT_COMPLETED_TRANSITION) + // Ignore-able transitions. + .addTransition(TaskStateInternal.SUCCEEDED, TaskStateInternal.SUCCEEDED, + EnumSet.of( + TaskEventType.T_ADD_SPEC_ATTEMPT, + TaskEventType.T_TERMINATE, + TaskEventType.T_ATTEMPT_LAUNCHED)) + .addTransition(TaskStateInternal.SUCCEEDED, TaskStateInternal.SUCCEEDED, + TaskEventType.T_SCHEDULE) + + // Transitions from FAILED state + .addTransition(TaskStateInternal.FAILED, TaskStateInternal.FAILED, + EnumSet.of( + TaskEventType.T_ATTEMPT_FAILED, + TaskEventType.T_ATTEMPT_KILLED, + TaskEventType.T_ATTEMPT_SUCCEEDED), REDUNDANT_COMPLETED_TRANSITION) + .addTransition(TaskStateInternal.FAILED, TaskStateInternal.FAILED, + EnumSet.of( + TaskEventType.T_TERMINATE, + TaskEventType.T_SCHEDULE, + TaskEventType.T_ADD_SPEC_ATTEMPT, + TaskEventType.T_ATTEMPT_LAUNCHED)) + + // Transitions from KILLED state + // Ignorable event: T_ATTEMPT_KILLED + // Refer to TEZ-2379 + // T_ATTEMPT_KILLED can show up in KILLED state as + // a SUCCEEDED attempt can still transition to KILLED after receiving + // a KILL event. + // This could happen when there is a race where the task receives a + // kill event, it tries to kill all running attempts and a potential + // running attempt succeeds before it receives the kill event. + // The task will then receive both a SUCCEEDED and KILLED + // event from the same attempt. + // Duplicate events from a single attempt in KILL_WAIT are handled + // properly. However, the subsequent T_ATTEMPT_KILLED event might + // be received after the task reaches its terminal KILLED state. + .addTransition(TaskStateInternal.KILLED, TaskStateInternal.KILLED, + EnumSet.of( + TaskEventType.T_TERMINATE, + TaskEventType.T_SCHEDULE, + TaskEventType.T_ADD_SPEC_ATTEMPT, + TaskEventType.T_ATTEMPT_LAUNCHED, + TaskEventType.T_ATTEMPT_SUCCEEDED, + TaskEventType.T_ATTEMPT_FAILED, + TaskEventType.T_ATTEMPT_KILLED)) + + // create the topology tables + .installTopology(); private void augmentStateMachine() { stateMachine @@ -318,7 +317,7 @@ private void augmentStateMachine() { } private final StateMachineTez - stateMachine; + stateMachine; // TODO: Recovery /* @@ -367,13 +366,13 @@ public TaskState getState() { } public TaskImpl(TezVertexID vertexId, int taskIndex, - EventHandler eventHandler, Configuration conf, - TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, - Clock clock, TaskHeartbeatHandler thh, AppContext appContext, - boolean leafVertex, Resource resource, - ContainerContext containerContext, - StateChangeNotifier stateChangeNotifier, - Vertex vertex) { + EventHandler eventHandler, Configuration conf, + TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, + Clock clock, TaskHeartbeatHandler thh, AppContext appContext, + boolean leafVertex, Resource resource, + ContainerContext containerContext, + StateChangeNotifier stateChangeNotifier, + Vertex vertex) { this.conf = conf; this.clock = clock; ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); @@ -398,7 +397,7 @@ public TaskImpl(TezVertexID vertexId, int taskIndex, stateMachineFactory.make(this), this); augmentStateMachine(); } - + @Override public Map getAttempts() { readLock.lock(); @@ -417,7 +416,7 @@ public Map getAttempts() { readLock.unlock(); } } - + @Override public TaskAttempt getAttempt(TezTaskAttemptID attemptID) { readLock.lock(); @@ -478,7 +477,7 @@ public TezCounters getCounters() { readLock.unlock(); } } - + TaskStatistics getStatistics() { // simply return the stats from the best attempt readLock.lock(); @@ -516,7 +515,7 @@ public float getProgress() { @Override public ArrayList getTaskAttemptTezEvents(TezTaskAttemptID attemptID, - int fromEventId, int maxEvents) { + int fromEventId, int maxEvents) { ArrayList events = EMPTY_TASK_ATTEMPT_TEZ_EVENTS; readLock.lock(); @@ -549,7 +548,7 @@ public ArrayList getTaskAttemptTezEvents(TezTaskAttemptID attemptID, readLock.unlock(); } } - + @Override public TaskSpec getBaseTaskSpec() { readLock.lock(); @@ -559,7 +558,7 @@ public TaskSpec getBaseTaskSpec() { readLock.unlock(); } } - + @Override public TaskLocationHint getTaskLocationHint() { readLock.lock(); @@ -667,11 +666,11 @@ private TaskAttempt selectBestAttempt() { for (TaskAttempt at : attempts.values()) { switch (at.getState()) { - // ignore all failed task attempts - case FAILED: - case KILLED: - continue; - default: + // ignore all failed task attempts + case FAILED: + case KILLED: + continue; + default: } if (result == null) { result = at; //The first time around @@ -698,7 +697,7 @@ public boolean canCommit(TezTaskAttemptID taskAttemptID) { // backlog to clear. returning false will make the attempt come back to us. LOG.info( "Event processing delay. " - + "Attempt committing before state machine transitioned to running : Task {}", taskId); + + "Attempt committing before state machine transitioned to running : Task {}", taskId); return false; } // at this point the attempt is no longer in scheduled state or else we would still @@ -739,7 +738,6 @@ public boolean canCommit(TezTaskAttemptID taskAttemptID) { LOG.debug("{} is current committer. Commit waiting for: {}", commitAttempt, taskAttemptID); return false; } - } finally { writeLock.unlock(); } @@ -879,15 +877,14 @@ protected void internalErrorUncaughtException(TaskEventType type, Exception e) { DAGEventType.INTERNAL_ERROR)); } - private void sendTaskAttemptCompletionEvent(TezTaskAttemptID attemptId, - TaskAttemptStateInternal attemptState) { + TaskAttemptStateInternal attemptState) { eventHandler.handle(new VertexEventTaskAttemptCompleted(attemptId, attemptState)); } // always called inside a transition, in turn inside the Write Lock private void handleTaskAttemptCompletion(TezTaskAttemptID attemptId, - TaskAttemptStateInternal attemptState) { + TaskAttemptStateInternal attemptState) { this.sendTaskAttemptCompletionEvent(attemptId, attemptState); sendDAGSchedulerFinishedEvent(attemptId); } @@ -897,21 +894,21 @@ private void sendDAGSchedulerFinishedEvent(TezTaskAttemptID taId) { eventHandler.handle(new DAGEventSchedulerUpdate( DAGEventSchedulerUpdate.UpdateType.TA_COMPLETED, attempts.get(taId))); } - + private static void unSucceed(TaskImpl task) { task.commitAttempt = null; task.successfulAttempt = null; } - + /** - * @return a String representation of the splits. - * - * Subclasses can override this method to provide their own representations - * of splits (if any). - * - */ - protected String getSplitsAsString(){ - return ""; + * @return a String representation of the splits. + * + * Subclasses can override this method to provide their own representations + * of splits (if any). + * + */ + protected String getSplitsAsString() { + return ""; } protected void logJobHistoryTaskStartedEvent() { @@ -937,7 +934,7 @@ protected void logJobHistoryTaskFailedEvent(TaskState finalState) { this.finishTime = clock.getTime(); TaskFinishedEvent finishEvt = new TaskFinishedEvent(taskId, getVertex().getName(), getLaunchTime(), this.finishTime, null, - finalState, + finalState, StringUtils.join(getDiagnostics(), LINE_SEPARATOR), getCounters(), failedAttempts); this.appContext.getHistoryHandler().handle( @@ -949,7 +946,7 @@ private void addDiagnosticInfo(String diag) { diagnostics.add(diag); } } - + @VisibleForTesting int getUncompletedAttemptsCount() { try { @@ -981,7 +978,7 @@ public boolean apply(Boolean state) { } private static class InitialScheduleTransition - implements MultipleArcTransition { + implements MultipleArcTransition { @Override public TaskStateInternal transition(TaskImpl task, TaskEvent event) { @@ -1026,7 +1023,7 @@ public TaskStateInternal transition(TaskImpl task, TaskEvent event) { // problems, so we can take later failures in parallel and flush the // job quickly when this happens. private static class RedundantScheduleTransition - implements SingleArcTransition { + implements SingleArcTransition { @Override public void transition(TaskImpl task, TaskEvent event) { @@ -1062,7 +1059,6 @@ public void transition(TaskImpl task, TaskEvent event) { } } - private static class AttemptSucceededTransition implements MultipleArcTransition { @@ -1089,7 +1085,7 @@ private String recoverSuccessTaskAttempt(TaskImpl task) { } try { committer.recoverTask(task.getTaskID().getId(), - task.appContext.getApplicationAttemptId().getAttemptId()-1); + task.appContext.getApplicationAttemptId().getAttemptId() - 1); } catch (Exception e) { errorMsg = "Task recovery failed by committer: " + ExceptionUtils.getStackTrace(e); @@ -1109,7 +1105,7 @@ public TaskStateInternal transition(TaskImpl task, TaskEvent event) { TezTaskAttemptID successTaId = ((TaskEventTAUpdate) event).getTaskAttemptID(); // Try to recover the succeeded TaskAttempt. It may be not recoverable if has committer which don't support // recovery. In that case just reschedule new attempt if numFailedAttempts does not exceeded maxFailedAttempts. - if (task.recoveryData!= null + if (task.recoveryData != null && task.recoveryData.isTaskAttemptSucceeded(successTaId)) { String errorMsg = recoverSuccessTaskAttempt(task); if (errorMsg != null) { @@ -1184,7 +1180,7 @@ public TaskStateInternal transition(TaskImpl task, TaskEvent event) { TaskEventTAUpdate castEvent = (TaskEventTAUpdate) event; task.addDiagnosticInfo("TaskAttempt " + castEvent.getTaskAttemptID().getId() + " killed"); - if (task.commitAttempt !=null && + if (task.commitAttempt != null && castEvent.getTaskAttemptID().equals(task.commitAttempt)) { task.commitAttempt = null; } @@ -1199,7 +1195,7 @@ public TaskStateInternal transition(TaskImpl task, TaskEvent event) { TaskEventTAKilled killEvent = (TaskEventTAKilled) event; if (killEvent.getCausalEvent() instanceof TaskAttemptEventTerminationCauseEvent) { TaskAttemptEventTerminationCauseEvent cause = - (TaskAttemptEventTerminationCauseEvent)killEvent.getCausalEvent(); + (TaskAttemptEventTerminationCauseEvent) killEvent.getCausalEvent(); isRejection = cause.getTerminationCause() == TaskAttemptTerminationCause.SERVICE_BUSY; } } @@ -1222,7 +1218,7 @@ private static class KillWaitAttemptCompletedTransition implements @Override public TaskStateInternal transition(TaskImpl task, TaskEvent event) { - TaskEventTAUpdate castEvent = (TaskEventTAUpdate)event; + TaskEventTAUpdate castEvent = (TaskEventTAUpdate) event; task.handleTaskAttemptCompletion( castEvent.getTaskAttemptID(), TaskAttemptStateInternal.KILLED); @@ -1238,17 +1234,17 @@ public TaskStateInternal transition(TaskImpl task, TaskEvent event) { return task.getInternalState(); } } - + private boolean shouldScheduleNewAttempt() { return (getUncompletedAttemptsCount() == 0 - && successfulAttempt == null); + && successfulAttempt == null); } private static class AttemptFailedTransition implements - MultipleArcTransition { + MultipleArcTransition { private TezTaskAttemptID schedulingCausalTA; - + @Override public TaskStateInternal transition(TaskImpl task, TaskEvent event) { task.failedAttempts++; @@ -1281,7 +1277,7 @@ public TaskStateInternal transition(TaskImpl task, TaskEvent event) { LOG.info("Scheduling new attempt for task: " + task.getTaskID() + ", currentFailedAttempts: " + task.failedAttempts + ", maxFailedAttempts: " + task.maxFailedAttempts + ", maxAttempts: " + task.maxAttempts); - if (!task.addAndScheduleAttempt(getSchedulingCausalTA())){ + if (!task.addAndScheduleAttempt(getSchedulingCausalTA())) { return task.finished(TaskStateInternal.FAILED); } } @@ -1309,7 +1305,7 @@ public TaskStateInternal transition(TaskImpl task, TaskEvent event) { protected TaskStateInternal getDefaultState(TaskImpl task) { return task.getInternalState(); } - + protected TezTaskAttemptID getSchedulingCausalTA() { return schedulingCausalTA; } @@ -1337,7 +1333,7 @@ public TaskStateInternal transition(TaskImpl task, TaskEvent event) { containerId, failedAttemptId, true)); } } - + if (task.getInternalState() == TaskStateInternal.SUCCEEDED && !failedAttemptId.equals(task.successfulAttempt)) { // don't allow a different task attempt to override a previous @@ -1352,7 +1348,7 @@ public TaskStateInternal transition(TaskImpl task, TaskEvent event) { task.internalError(event.getType()); } Preconditions.checkState(castEvent.getCausalEvent() != null); - TaskAttemptEventOutputFailed destinationEvent = + TaskAttemptEventOutputFailed destinationEvent = (TaskAttemptEventOutputFailed) castEvent.getCausalEvent(); schedulingCausalTA = destinationEvent.getInputFailedEvent().getSourceInfo().getTaskAttemptID(); @@ -1371,7 +1367,7 @@ public TaskStateInternal transition(TaskImpl task, TaskEvent event) { return returnState; } - + @Override protected TezTaskAttemptID getSchedulingCausalTA() { return schedulingCausalTA; @@ -1384,7 +1380,7 @@ protected TaskStateInternal getDefaultState(TaskImpl task) { } private static class TaskRetroactiveKilledTransition implements - MultipleArcTransition { + MultipleArcTransition { @Override public TaskStateInternal transition(TaskImpl task, TaskEvent event) { @@ -1392,7 +1388,7 @@ public TaskStateInternal transition(TaskImpl task, TaskEvent event) { TaskEventTAUpdate attemptEvent = (TaskEventTAUpdate) event; TezTaskAttemptID attemptId = attemptEvent.getTaskAttemptID(); TaskStateInternal resultState = TaskStateInternal.SUCCEEDED; - if(task.successfulAttempt.equals(attemptId)) { + if (task.successfulAttempt.equals(attemptId)) { // typically we are here because this map task was run on a bad node and // we want to reschedule it on a different node. // Depending on whether there are previous failed attempts or not this @@ -1411,10 +1407,10 @@ public TaskStateInternal transition(TaskImpl task, TaskEvent event) { } private static class KillNewTransition - implements SingleArcTransition { + implements SingleArcTransition { @Override public void transition(TaskImpl task, TaskEvent event) { - TaskEventTermination terminateEvent = (TaskEventTermination)event; + TaskEventTermination terminateEvent = (TaskEventTermination) event; task.addDiagnosticInfo(terminateEvent.getDiagnosticInfo()); if (terminateEvent.isFromRecovery()) { if (LOG.isDebugEnabled()) { @@ -1434,7 +1430,7 @@ private static class AttemptRedundantCompletedTransition implements SingleArcTransition { @Override public void transition(TaskImpl task, TaskEvent event) { - TezTaskAttemptID successTaId = ((TaskEventTAUpdate)event).getTaskAttemptID(); + TezTaskAttemptID successTaId = ((TaskEventTAUpdate) event).getTaskAttemptID(); task.taskAttemptStatus.put(successTaId.getId(), true); } } @@ -1492,10 +1488,10 @@ public long getFirstAttemptStartTime() { } private static class KillTransition - implements SingleArcTransition { + implements SingleArcTransition { @Override public void transition(TaskImpl task, TaskEvent event) { - TaskEventTermination terminateEvent = (TaskEventTermination)event; + TaskEventTermination terminateEvent = (TaskEventTermination) event; task.addDiagnosticInfo(terminateEvent.getDiagnosticInfo()); // issue kill to all non finished attempts for (TaskAttempt attempt : task.attempts.values()) { @@ -1527,5 +1523,4 @@ void setCounters(TezCounters counters) { writeLock.unlock(); } } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskReportImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskReportImpl.java index 32ed94a5e0..9175b5298b 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskReportImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskReportImpl.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,7 +34,7 @@ public TaskReportImpl() { } public TaskReportImpl(TezTaskID taskID, TaskState taskState, - float progress, long startTime, long finishTime) { + float progress, long startTime, long finishTime) { this.taskID = taskID; this.taskState = taskState; this.progress = progress; @@ -91,5 +91,4 @@ public void setStartTime(long startTime) { public void setFinishTime(long finishTime) { this.finishTime = finishTime; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TezRootInputInitializerContextImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TezRootInputInitializerContextImpl.java index be4ee6068e..4070d61133 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TezRootInputInitializerContextImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TezRootInputInitializerContextImpl.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,8 +18,6 @@ package org.apache.tez.dag.app.dag.impl; - - import java.util.Set; import java.util.Objects; @@ -45,7 +43,6 @@ public class TezRootInputInitializerContextImpl implements private final AppContext appContext; private final RootInputInitializerManager manager; - // TODO Add support for counters - merged with the Vertex counters. public TezRootInputInitializerContextImpl( @@ -81,7 +78,7 @@ public String getInputName() { public UserPayload getInputUserPayload() { return this.input.getIODescriptor().getUserPayload(); } - + @Override public UserPayload getUserPayload() { return this.input.getControllerDescriptor().getUserPayload(); @@ -92,7 +89,7 @@ public Configuration getVertexConfiguration() { return vertex.getConf(); } - @Override + @Override public int getNumTasks() { return vertex.getTotalTasks(); } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java index e55b10a659..9cf979da50 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java @@ -1,19 +1,19 @@ /* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.impl; @@ -209,10 +209,11 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -/** Implementation of Vertex interface. Maintains the state machines of Vertex. +/** + * Implementation of Vertex interface. Maintains the state machines of Vertex. * The read and write calls use ReadWriteLock for concurrency. */ -@SuppressWarnings({ "rawtypes", "unchecked" }) +@SuppressWarnings({"rawtypes", "unchecked"}) public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandler { private static final String LINE_SEPARATOR = System @@ -256,7 +257,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandl // Useful when trying to serialize only the diff from global configs @VisibleForTesting Configuration vertexOnlyConf; - + private final boolean isSpeculationEnabled; @VisibleForTesting @@ -294,7 +295,7 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandl private final List diagnostics = new ArrayList(); protected final StateChangeNotifier stateChangeNotifier; - + //task/attempt related datastructures @VisibleForTesting int numSuccessSourceAttemptCompletions = 0; @@ -308,13 +309,13 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandl ROUTE_EVENT_TRANSITION = new RouteEventTransition(); private static final TaskAttemptCompletedEventTransition TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION = - new TaskAttemptCompletedEventTransition(); + new TaskAttemptCompletedEventTransition(); private static final SourceTaskAttemptCompletedEventTransition SOURCE_TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION = - new SourceTaskAttemptCompletedEventTransition(); + new SourceTaskAttemptCompletedEventTransition(); private static final CommitCompletedTransition COMMIT_COMPLETED_TRANSITION = - new CommitCompletedTransition(); + new CommitCompletedTransition(); private static final VertexStateChangedCallback STATE_CHANGED_CALLBACK = new VertexStateChangedCallback(); @@ -332,343 +333,342 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex, EventHandl Map> commitFutures = new ConcurrentHashMap>(); protected static final - StateMachineFactory - stateMachineFactory - = new StateMachineFactory - (VertexState.NEW) - - // Transitions from NEW state - .addTransition - (VertexState.NEW, - EnumSet.of(VertexState.NEW, VertexState.INITED, - VertexState.INITIALIZING, VertexState.FAILED, VertexState.KILLED), - VertexEventType.V_INIT, - new InitTransition()) - .addTransition(VertexState.NEW, - EnumSet.of(VertexState.NEW), - VertexEventType.V_NULL_EDGE_INITIALIZED, - new NullEdgeInitializedTransition()) - .addTransition(VertexState.NEW, - EnumSet.of(VertexState.NEW), - VertexEventType.V_ROUTE_EVENT, - ROUTE_EVENT_TRANSITION) - .addTransition(VertexState.NEW, - EnumSet.of(VertexState.NEW), - VertexEventType.V_SOURCE_TASK_ATTEMPT_COMPLETED, - SOURCE_TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION) - .addTransition - (VertexState.NEW, - EnumSet.of(VertexState.NEW, - VertexState.SUCCEEDED, VertexState.FAILED, - VertexState.KILLED, VertexState.ERROR), - VertexEventType.V_RECOVER, - new RecoverTransition()) - .addTransition(VertexState.NEW, VertexState.NEW, - VertexEventType.V_SOURCE_VERTEX_STARTED, - new SourceVertexStartedTransition()) - .addTransition(VertexState.NEW, VertexState.KILLED, - VertexEventType.V_TERMINATE, - new TerminateNewVertexTransition()) - .addTransition(VertexState.NEW, VertexState.ERROR, - VertexEventType.V_INTERNAL_ERROR, - INTERNAL_ERROR_TRANSITION) - - // Transitions from INITIALIZING state - .addTransition(VertexState.INITIALIZING, - EnumSet.of(VertexState.INITIALIZING, VertexState.INITED, - VertexState.FAILED), - VertexEventType.V_ROOT_INPUT_INITIALIZED, - new RootInputInitializedTransition()) - .addTransition(VertexState.INITIALIZING, - EnumSet.of(VertexState.INITIALIZING, VertexState.INITED, - VertexState.FAILED), - VertexEventType.V_INPUT_DATA_INFORMATION, - new InputDataInformationTransition()) - .addTransition(VertexState.INITIALIZING, - EnumSet.of(VertexState.INITED, VertexState.FAILED), - VertexEventType.V_READY_TO_INIT, - new VertexInitializedTransition()) - .addTransition(VertexState.INITIALIZING, - EnumSet.of(VertexState.FAILED), + StateMachineFactory + stateMachineFactory + = new StateMachineFactory + (VertexState.NEW) + + // Transitions from NEW state + .addTransition + (VertexState.NEW, + EnumSet.of(VertexState.NEW, VertexState.INITED, + VertexState.INITIALIZING, VertexState.FAILED, VertexState.KILLED), + VertexEventType.V_INIT, + new InitTransition()) + .addTransition(VertexState.NEW, + EnumSet.of(VertexState.NEW), + VertexEventType.V_NULL_EDGE_INITIALIZED, + new NullEdgeInitializedTransition()) + .addTransition(VertexState.NEW, + EnumSet.of(VertexState.NEW), + VertexEventType.V_ROUTE_EVENT, + ROUTE_EVENT_TRANSITION) + .addTransition(VertexState.NEW, + EnumSet.of(VertexState.NEW), + VertexEventType.V_SOURCE_TASK_ATTEMPT_COMPLETED, + SOURCE_TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION) + .addTransition + (VertexState.NEW, + EnumSet.of(VertexState.NEW, + VertexState.SUCCEEDED, VertexState.FAILED, + VertexState.KILLED, VertexState.ERROR), + VertexEventType.V_RECOVER, + new RecoverTransition()) + .addTransition(VertexState.NEW, VertexState.NEW, + VertexEventType.V_SOURCE_VERTEX_STARTED, + new SourceVertexStartedTransition()) + .addTransition(VertexState.NEW, VertexState.KILLED, + VertexEventType.V_TERMINATE, + new TerminateNewVertexTransition()) + .addTransition(VertexState.NEW, VertexState.ERROR, + VertexEventType.V_INTERNAL_ERROR, + INTERNAL_ERROR_TRANSITION) + + // Transitions from INITIALIZING state + .addTransition(VertexState.INITIALIZING, + EnumSet.of(VertexState.INITIALIZING, VertexState.INITED, + VertexState.FAILED), + VertexEventType.V_ROOT_INPUT_INITIALIZED, + new RootInputInitializedTransition()) + .addTransition(VertexState.INITIALIZING, + EnumSet.of(VertexState.INITIALIZING, VertexState.INITED, + VertexState.FAILED), + VertexEventType.V_INPUT_DATA_INFORMATION, + new InputDataInformationTransition()) + .addTransition(VertexState.INITIALIZING, + EnumSet.of(VertexState.INITED, VertexState.FAILED), + VertexEventType.V_READY_TO_INIT, + new VertexInitializedTransition()) + .addTransition(VertexState.INITIALIZING, + EnumSet.of(VertexState.FAILED), + VertexEventType.V_ROOT_INPUT_FAILED, + new RootInputInitFailedTransition()) + .addTransition(VertexState.INITIALIZING, VertexState.INITIALIZING, + VertexEventType.V_START, + new StartWhileInitializingTransition()) + .addTransition(VertexState.INITIALIZING, VertexState.INITIALIZING, + VertexEventType.V_SOURCE_VERTEX_STARTED, + new SourceVertexStartedTransition()) + .addTransition(VertexState.INITIALIZING, + EnumSet.of(VertexState.INITIALIZING), + VertexEventType.V_SOURCE_TASK_ATTEMPT_COMPLETED, + SOURCE_TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION) + .addTransition(VertexState.INITIALIZING, + EnumSet.of(VertexState.INITIALIZING, VertexState.FAILED), + VertexEventType.V_ROUTE_EVENT, + ROUTE_EVENT_TRANSITION) + .addTransition(VertexState.INITIALIZING, EnumSet.of(VertexState.FAILED), + VertexEventType.V_MANAGER_USER_CODE_ERROR, + new VertexManagerUserCodeErrorTransition()) + .addTransition(VertexState.INITIALIZING, VertexState.KILLED, + VertexEventType.V_TERMINATE, + new TerminateInitingVertexTransition()) + .addTransition(VertexState.INITIALIZING, VertexState.ERROR, + VertexEventType.V_INTERNAL_ERROR, + INTERNAL_ERROR_TRANSITION) + .addTransition(VertexState.INITIALIZING, + EnumSet.of(VertexState.INITIALIZING, VertexState.INITED, + VertexState.FAILED), + VertexEventType.V_NULL_EDGE_INITIALIZED, + new NullEdgeInitializedTransition()) + + // Transitions from INITED state + // SOURCE_VERTEX_STARTED - for sources which determine parallelism, + // they must complete before this vertex can start. + .addTransition(VertexState.INITED, + EnumSet.of(VertexState.FAILED), + VertexEventType.V_ROOT_INPUT_FAILED, + new RootInputInitFailedTransition()) + .addTransition(VertexState.INITED, VertexState.INITED, + VertexEventType.V_SOURCE_VERTEX_STARTED, + new SourceVertexStartedTransition()) + .addTransition(VertexState.INITED, + EnumSet.of(VertexState.INITED), + VertexEventType.V_SOURCE_TASK_ATTEMPT_COMPLETED, + SOURCE_TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION) + .addTransition(VertexState.INITED, + EnumSet.of(VertexState.RUNNING, VertexState.INITED, VertexState.TERMINATING), + VertexEventType.V_START, + new StartTransition()) + .addTransition(VertexState.INITED, + EnumSet.of(VertexState.INITED, VertexState.FAILED), + VertexEventType.V_ROUTE_EVENT, + ROUTE_EVENT_TRANSITION) + .addTransition(VertexState.INITED, VertexState.KILLED, + VertexEventType.V_TERMINATE, + new TerminateInitedVertexTransition()) + .addTransition(VertexState.INITED, EnumSet.of(VertexState.FAILED), + VertexEventType.V_MANAGER_USER_CODE_ERROR, + new VertexManagerUserCodeErrorTransition()) + .addTransition(VertexState.INITED, VertexState.ERROR, + VertexEventType.V_INTERNAL_ERROR, + INTERNAL_ERROR_TRANSITION) + + // Transitions from RUNNING state + .addTransition(VertexState.RUNNING, + EnumSet.of(VertexState.TERMINATING), + VertexEventType.V_ROOT_INPUT_FAILED, + new RootInputInitFailedTransition()) + .addTransition(VertexState.RUNNING, VertexState.RUNNING, + VertexEventType.V_TASK_ATTEMPT_COMPLETED, + TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION) + .addTransition(VertexState.RUNNING, + EnumSet.of(VertexState.RUNNING, VertexState.TERMINATING), + VertexEventType.V_SOURCE_TASK_ATTEMPT_COMPLETED, + SOURCE_TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION) + .addTransition + (VertexState.RUNNING, + EnumSet.of(VertexState.RUNNING, + VertexState.COMMITTING, + VertexState.SUCCEEDED, VertexState.TERMINATING, VertexState.FAILED, + VertexState.ERROR), + VertexEventType.V_TASK_COMPLETED, + new TaskCompletedTransition()) + .addTransition(VertexState.RUNNING, VertexState.TERMINATING, + VertexEventType.V_TERMINATE, + new VertexKilledTransition()) + .addTransition(VertexState.RUNNING, EnumSet.of(VertexState.TERMINATING), + VertexEventType.V_MANAGER_USER_CODE_ERROR, + new VertexManagerUserCodeErrorTransition()) + .addTransition(VertexState.RUNNING, VertexState.RUNNING, + VertexEventType.V_TASK_RESCHEDULED, + new TaskRescheduledTransition()) + .addTransition(VertexState.RUNNING, + EnumSet.of(VertexState.RUNNING, VertexState.SUCCEEDED, + VertexState.FAILED), + VertexEventType.V_COMPLETED, + new VertexNoTasksCompletedTransition()) + .addTransition( + VertexState.RUNNING, + VertexState.ERROR, VertexEventType.V_INTERNAL_ERROR, + INTERNAL_ERROR_TRANSITION) + .addTransition( + VertexState.RUNNING, + EnumSet.of(VertexState.RUNNING, VertexState.TERMINATING), + VertexEventType.V_ROUTE_EVENT, + ROUTE_EVENT_TRANSITION) + + // Transitions from COMMITTING state. + .addTransition( + VertexState.COMMITTING, + EnumSet.of(VertexState.COMMITTING, VertexState.TERMINATING, + VertexState.SUCCEEDED, VertexState.FAILED), + VertexEventType.V_COMMIT_COMPLETED, + COMMIT_COMPLETED_TRANSITION) + .addTransition( + VertexState.COMMITTING, + VertexState.TERMINATING, + VertexEventType.V_TERMINATE, + new VertexKilledWhileCommittingTransition()) + .addTransition( + VertexState.COMMITTING, + VertexState.ERROR, + VertexEventType.V_INTERNAL_ERROR, + INTERNAL_ERROR_TRANSITION) + .addTransition( + VertexState.COMMITTING, + EnumSet.of(VertexState.COMMITTING, VertexState.TERMINATING), + VertexEventType.V_ROUTE_EVENT, + ROUTE_EVENT_TRANSITION) + .addTransition( + VertexState.COMMITTING, + VertexState.TERMINATING, + VertexEventType.V_TASK_RESCHEDULED, + new TaskRescheduledWhileCommittingTransition()) + .addTransition(VertexState.COMMITTING, + EnumSet.of(VertexState.TERMINATING), + VertexEventType.V_MANAGER_USER_CODE_ERROR, + new VertexManagerUserCodeErrorTransition()) + + // Transitions from TERMINATING state. + .addTransition + (VertexState.TERMINATING, + EnumSet.of(VertexState.TERMINATING, VertexState.KILLED, VertexState.FAILED, VertexState.ERROR), + VertexEventType.V_TASK_COMPLETED, + new TaskCompletedTransition()) + .addTransition + (VertexState.TERMINATING, + EnumSet.of(VertexState.TERMINATING, VertexState.KILLED, VertexState.FAILED, VertexState.ERROR), + VertexEventType.V_COMPLETED, + new VertexNoTasksCompletedTransition()) + .addTransition( + VertexState.TERMINATING, + VertexState.ERROR, VertexEventType.V_INTERNAL_ERROR, + INTERNAL_ERROR_TRANSITION) + .addTransition( + VertexState.TERMINATING, + EnumSet.of(VertexState.TERMINATING, VertexState.FAILED, VertexState.KILLED, VertexState.ERROR), + VertexEventType.V_COMMIT_COMPLETED, + COMMIT_COMPLETED_TRANSITION) + // Ignore-able events + .addTransition(VertexState.TERMINATING, VertexState.TERMINATING, + EnumSet.of(VertexEventType.V_TERMINATE, + VertexEventType.V_MANAGER_USER_CODE_ERROR, VertexEventType.V_ROOT_INPUT_FAILED, - new RootInputInitFailedTransition()) - .addTransition(VertexState.INITIALIZING, VertexState.INITIALIZING, - VertexEventType.V_START, - new StartWhileInitializingTransition()) - .addTransition(VertexState.INITIALIZING, VertexState.INITIALIZING, VertexEventType.V_SOURCE_VERTEX_STARTED, - new SourceVertexStartedTransition()) - .addTransition(VertexState.INITIALIZING, - EnumSet.of(VertexState.INITIALIZING), - VertexEventType.V_SOURCE_TASK_ATTEMPT_COMPLETED, - SOURCE_TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION) - .addTransition(VertexState.INITIALIZING, - EnumSet.of(VertexState.INITIALIZING, VertexState.FAILED), + VertexEventType.V_ROOT_INPUT_INITIALIZED, + VertexEventType.V_NULL_EDGE_INITIALIZED, VertexEventType.V_ROUTE_EVENT, - ROUTE_EVENT_TRANSITION) - .addTransition(VertexState.INITIALIZING, EnumSet.of(VertexState.FAILED), + VertexEventType.V_SOURCE_TASK_ATTEMPT_COMPLETED, + VertexEventType.V_TASK_ATTEMPT_COMPLETED, + VertexEventType.V_TASK_RESCHEDULED, + VertexEventType.V_DELETE_SHUFFLE_DATA)) + + // Transitions from SUCCEEDED state + .addTransition( + VertexState.SUCCEEDED, + VertexState.ERROR, VertexEventType.V_INTERNAL_ERROR, + INTERNAL_ERROR_TRANSITION) + .addTransition(VertexState.SUCCEEDED, + EnumSet.of(VertexState.RUNNING, VertexState.FAILED), + VertexEventType.V_TASK_RESCHEDULED, + new TaskRescheduledAfterVertexSuccessTransition()) + + .addTransition( + VertexState.SUCCEEDED, + EnumSet.of(VertexState.SUCCEEDED, VertexState.FAILED), + // accumulate these in case we get restarted + VertexEventType.V_ROUTE_EVENT, + ROUTE_EVENT_TRANSITION) + .addTransition( + VertexState.SUCCEEDED, + EnumSet.of(VertexState.FAILED, VertexState.ERROR), + VertexEventType.V_TASK_COMPLETED, + new TaskCompletedAfterVertexSuccessTransition()) + // Ignore-able events + .addTransition(VertexState.SUCCEEDED, VertexState.SUCCEEDED, + EnumSet.of(VertexEventType.V_TERMINATE, + VertexEventType.V_ROOT_INPUT_FAILED, + VertexEventType.V_TASK_ATTEMPT_COMPLETED, + // after we are done reruns of source tasks should not affect + // us. These reruns may be triggered by other consumer vertices. + // We should have been in RUNNING state if we had triggered the + // reruns. + VertexEventType.V_SOURCE_TASK_ATTEMPT_COMPLETED)) + .addTransition(VertexState.SUCCEEDED, VertexState.SUCCEEDED, + VertexEventType.V_TASK_ATTEMPT_COMPLETED, + new TaskAttemptCompletedEventTransition()) + .addTransition(VertexState.SUCCEEDED, VertexState.SUCCEEDED, + VertexEventType.V_DELETE_SHUFFLE_DATA, + new VertexShuffleDeleteTransition()) + + // Transitions from FAILED state + .addTransition( + VertexState.FAILED, + VertexState.ERROR, VertexEventType.V_INTERNAL_ERROR, + INTERNAL_ERROR_TRANSITION) + // Ignore-able events + .addTransition(VertexState.FAILED, VertexState.FAILED, + EnumSet.of(VertexEventType.V_TERMINATE, VertexEventType.V_MANAGER_USER_CODE_ERROR, - new VertexManagerUserCodeErrorTransition()) - .addTransition(VertexState.INITIALIZING, VertexState.KILLED, - VertexEventType.V_TERMINATE, - new TerminateInitingVertexTransition()) - .addTransition(VertexState.INITIALIZING, VertexState.ERROR, - VertexEventType.V_INTERNAL_ERROR, - INTERNAL_ERROR_TRANSITION) - .addTransition(VertexState.INITIALIZING, - EnumSet.of(VertexState.INITIALIZING, VertexState.INITED, - VertexState.FAILED), - VertexEventType.V_NULL_EDGE_INITIALIZED, - new NullEdgeInitializedTransition()) - - // Transitions from INITED state - // SOURCE_VERTEX_STARTED - for sources which determine parallelism, - // they must complete before this vertex can start. - .addTransition(VertexState.INITED, - EnumSet.of(VertexState.FAILED), VertexEventType.V_ROOT_INPUT_FAILED, - new RootInputInitFailedTransition()) - .addTransition(VertexState.INITED, VertexState.INITED, VertexEventType.V_SOURCE_VERTEX_STARTED, - new SourceVertexStartedTransition()) - .addTransition(VertexState.INITED, - EnumSet.of(VertexState.INITED), - VertexEventType.V_SOURCE_TASK_ATTEMPT_COMPLETED, - SOURCE_TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION) - .addTransition(VertexState.INITED, - EnumSet.of(VertexState.RUNNING, VertexState.INITED, VertexState.TERMINATING), + VertexEventType.V_TASK_RESCHEDULED, VertexEventType.V_START, - new StartTransition()) - .addTransition(VertexState.INITED, - EnumSet.of(VertexState.INITED, VertexState.FAILED), VertexEventType.V_ROUTE_EVENT, - ROUTE_EVENT_TRANSITION) - .addTransition(VertexState.INITED, VertexState.KILLED, - VertexEventType.V_TERMINATE, - new TerminateInitedVertexTransition()) - .addTransition(VertexState.INITED, EnumSet.of(VertexState.FAILED), - VertexEventType.V_MANAGER_USER_CODE_ERROR, - new VertexManagerUserCodeErrorTransition()) - .addTransition(VertexState.INITED, VertexState.ERROR, - VertexEventType.V_INTERNAL_ERROR, - INTERNAL_ERROR_TRANSITION) - - // Transitions from RUNNING state - .addTransition(VertexState.RUNNING, - EnumSet.of(VertexState.TERMINATING), - VertexEventType.V_ROOT_INPUT_FAILED, - new RootInputInitFailedTransition()) - .addTransition(VertexState.RUNNING, VertexState.RUNNING, VertexEventType.V_TASK_ATTEMPT_COMPLETED, - TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION) - .addTransition(VertexState.RUNNING, - EnumSet.of(VertexState.RUNNING, VertexState.TERMINATING), - VertexEventType.V_SOURCE_TASK_ATTEMPT_COMPLETED, - SOURCE_TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION) - .addTransition - (VertexState.RUNNING, - EnumSet.of(VertexState.RUNNING, - VertexState.COMMITTING, - VertexState.SUCCEEDED, VertexState.TERMINATING, VertexState.FAILED, - VertexState.ERROR), VertexEventType.V_TASK_COMPLETED, - new TaskCompletedTransition()) - .addTransition(VertexState.RUNNING, VertexState.TERMINATING, - VertexEventType.V_TERMINATE, - new VertexKilledTransition()) - .addTransition(VertexState.RUNNING, EnumSet.of(VertexState.TERMINATING), + VertexEventType.V_ROOT_INPUT_INITIALIZED, + VertexEventType.V_SOURCE_TASK_ATTEMPT_COMPLETED, + VertexEventType.V_NULL_EDGE_INITIALIZED, + VertexEventType.V_INPUT_DATA_INFORMATION, + VertexEventType.V_DELETE_SHUFFLE_DATA)) + + // Transitions from KILLED state + .addTransition( + VertexState.KILLED, + VertexState.ERROR, VertexEventType.V_INTERNAL_ERROR, + INTERNAL_ERROR_TRANSITION) + // Ignore-able events + .addTransition(VertexState.KILLED, VertexState.KILLED, + EnumSet.of(VertexEventType.V_TERMINATE, VertexEventType.V_MANAGER_USER_CODE_ERROR, - new VertexManagerUserCodeErrorTransition()) - .addTransition(VertexState.RUNNING, VertexState.RUNNING, - VertexEventType.V_TASK_RESCHEDULED, - new TaskRescheduledTransition()) - .addTransition(VertexState.RUNNING, - EnumSet.of(VertexState.RUNNING, VertexState.SUCCEEDED, - VertexState.FAILED), - VertexEventType.V_COMPLETED, - new VertexNoTasksCompletedTransition()) - .addTransition( - VertexState.RUNNING, - VertexState.ERROR, VertexEventType.V_INTERNAL_ERROR, - INTERNAL_ERROR_TRANSITION) - .addTransition( - VertexState.RUNNING, - EnumSet.of(VertexState.RUNNING, VertexState.TERMINATING), - VertexEventType.V_ROUTE_EVENT, - ROUTE_EVENT_TRANSITION) - - // Transitions from COMMITTING state. - .addTransition( - VertexState.COMMITTING, - EnumSet.of(VertexState.COMMITTING, VertexState.TERMINATING, - VertexState.SUCCEEDED, VertexState.FAILED), - VertexEventType.V_COMMIT_COMPLETED, - COMMIT_COMPLETED_TRANSITION) - .addTransition( - VertexState.COMMITTING, - VertexState.TERMINATING, - VertexEventType.V_TERMINATE, - new VertexKilledWhileCommittingTransition()) - .addTransition( - VertexState.COMMITTING, - VertexState.ERROR, - VertexEventType.V_INTERNAL_ERROR, - INTERNAL_ERROR_TRANSITION) - .addTransition( - VertexState.COMMITTING, - EnumSet.of(VertexState.COMMITTING, VertexState.TERMINATING), + VertexEventType.V_ROOT_INPUT_FAILED, + VertexEventType.V_INIT, + VertexEventType.V_SOURCE_VERTEX_STARTED, + VertexEventType.V_START, VertexEventType.V_ROUTE_EVENT, - ROUTE_EVENT_TRANSITION) - .addTransition( - VertexState.COMMITTING, - VertexState.TERMINATING, VertexEventType.V_TASK_RESCHEDULED, - new TaskRescheduledWhileCommittingTransition()) - .addTransition(VertexState.COMMITTING, - EnumSet.of(VertexState.TERMINATING), - VertexEventType.V_MANAGER_USER_CODE_ERROR, - new VertexManagerUserCodeErrorTransition()) - - // Transitions from TERMINATING state. - .addTransition - (VertexState.TERMINATING, - EnumSet.of(VertexState.TERMINATING, VertexState.KILLED, VertexState.FAILED, VertexState.ERROR), + VertexEventType.V_TASK_ATTEMPT_COMPLETED, + VertexEventType.V_SOURCE_TASK_ATTEMPT_COMPLETED, VertexEventType.V_TASK_COMPLETED, - new TaskCompletedTransition()) - .addTransition - (VertexState.TERMINATING, - EnumSet.of(VertexState.TERMINATING, VertexState.KILLED, VertexState.FAILED, VertexState.ERROR), - VertexEventType.V_COMPLETED, - new VertexNoTasksCompletedTransition()) - .addTransition( - VertexState.TERMINATING, - VertexState.ERROR, VertexEventType.V_INTERNAL_ERROR, - INTERNAL_ERROR_TRANSITION) - .addTransition( - VertexState.TERMINATING, - EnumSet.of(VertexState.TERMINATING, VertexState.FAILED, VertexState.KILLED, VertexState.ERROR), - VertexEventType.V_COMMIT_COMPLETED, - COMMIT_COMPLETED_TRANSITION) - // Ignore-able events - .addTransition(VertexState.TERMINATING, VertexState.TERMINATING, - EnumSet.of(VertexEventType.V_TERMINATE, - VertexEventType.V_MANAGER_USER_CODE_ERROR, - VertexEventType.V_ROOT_INPUT_FAILED, - VertexEventType.V_SOURCE_VERTEX_STARTED, - VertexEventType.V_ROOT_INPUT_INITIALIZED, - VertexEventType.V_NULL_EDGE_INITIALIZED, - VertexEventType.V_ROUTE_EVENT, - VertexEventType.V_SOURCE_TASK_ATTEMPT_COMPLETED, - VertexEventType.V_TASK_ATTEMPT_COMPLETED, - VertexEventType.V_TASK_RESCHEDULED, - VertexEventType.V_DELETE_SHUFFLE_DATA)) - - // Transitions from SUCCEEDED state - .addTransition( - VertexState.SUCCEEDED, - VertexState.ERROR, VertexEventType.V_INTERNAL_ERROR, - INTERNAL_ERROR_TRANSITION) - .addTransition(VertexState.SUCCEEDED, - EnumSet.of(VertexState.RUNNING, VertexState.FAILED), - VertexEventType.V_TASK_RESCHEDULED, - new TaskRescheduledAfterVertexSuccessTransition()) + VertexEventType.V_ROOT_INPUT_INITIALIZED, + VertexEventType.V_NULL_EDGE_INITIALIZED, + VertexEventType.V_INPUT_DATA_INFORMATION, + VertexEventType.V_DELETE_SHUFFLE_DATA)) - .addTransition( - VertexState.SUCCEEDED, - EnumSet.of(VertexState.SUCCEEDED, VertexState.FAILED), - // accumulate these in case we get restarted + // No transitions from INTERNAL_ERROR state. Ignore all. + .addTransition( + VertexState.ERROR, + VertexState.ERROR, + EnumSet.of(VertexEventType.V_INIT, + VertexEventType.V_ROOT_INPUT_FAILED, + VertexEventType.V_SOURCE_VERTEX_STARTED, + VertexEventType.V_START, VertexEventType.V_ROUTE_EVENT, - ROUTE_EVENT_TRANSITION) - .addTransition( - VertexState.SUCCEEDED, - EnumSet.of(VertexState.FAILED, VertexState.ERROR), + VertexEventType.V_TERMINATE, + VertexEventType.V_MANAGER_USER_CODE_ERROR, VertexEventType.V_TASK_COMPLETED, - new TaskCompletedAfterVertexSuccessTransition()) - // Ignore-able events - .addTransition(VertexState.SUCCEEDED, VertexState.SUCCEEDED, - EnumSet.of(VertexEventType.V_TERMINATE, - VertexEventType.V_ROOT_INPUT_FAILED, - VertexEventType.V_TASK_ATTEMPT_COMPLETED, - // after we are done reruns of source tasks should not affect - // us. These reruns may be triggered by other consumer vertices. - // We should have been in RUNNING state if we had triggered the - // reruns. - VertexEventType.V_SOURCE_TASK_ATTEMPT_COMPLETED)) - .addTransition(VertexState.SUCCEEDED, VertexState.SUCCEEDED, VertexEventType.V_TASK_ATTEMPT_COMPLETED, - new TaskAttemptCompletedEventTransition()) - .addTransition(VertexState.SUCCEEDED, VertexState.SUCCEEDED, - VertexEventType.V_DELETE_SHUFFLE_DATA, - new VertexShuffleDeleteTransition()) - - - // Transitions from FAILED state - .addTransition( - VertexState.FAILED, - VertexState.ERROR, VertexEventType.V_INTERNAL_ERROR, - INTERNAL_ERROR_TRANSITION) - // Ignore-able events - .addTransition(VertexState.FAILED, VertexState.FAILED, - EnumSet.of(VertexEventType.V_TERMINATE, - VertexEventType.V_MANAGER_USER_CODE_ERROR, - VertexEventType.V_ROOT_INPUT_FAILED, - VertexEventType.V_SOURCE_VERTEX_STARTED, - VertexEventType.V_TASK_RESCHEDULED, - VertexEventType.V_START, - VertexEventType.V_ROUTE_EVENT, - VertexEventType.V_TASK_ATTEMPT_COMPLETED, - VertexEventType.V_TASK_COMPLETED, - VertexEventType.V_ROOT_INPUT_INITIALIZED, - VertexEventType.V_SOURCE_TASK_ATTEMPT_COMPLETED, - VertexEventType.V_NULL_EDGE_INITIALIZED, - VertexEventType.V_INPUT_DATA_INFORMATION, - VertexEventType.V_DELETE_SHUFFLE_DATA)) - - // Transitions from KILLED state - .addTransition( - VertexState.KILLED, - VertexState.ERROR, VertexEventType.V_INTERNAL_ERROR, - INTERNAL_ERROR_TRANSITION) - // Ignore-able events - .addTransition(VertexState.KILLED, VertexState.KILLED, - EnumSet.of(VertexEventType.V_TERMINATE, - VertexEventType.V_MANAGER_USER_CODE_ERROR, - VertexEventType.V_ROOT_INPUT_FAILED, - VertexEventType.V_INIT, - VertexEventType.V_SOURCE_VERTEX_STARTED, - VertexEventType.V_START, - VertexEventType.V_ROUTE_EVENT, - VertexEventType.V_TASK_RESCHEDULED, - VertexEventType.V_TASK_ATTEMPT_COMPLETED, - VertexEventType.V_SOURCE_TASK_ATTEMPT_COMPLETED, - VertexEventType.V_TASK_COMPLETED, - VertexEventType.V_ROOT_INPUT_INITIALIZED, - VertexEventType.V_NULL_EDGE_INITIALIZED, - VertexEventType.V_INPUT_DATA_INFORMATION, - VertexEventType.V_DELETE_SHUFFLE_DATA)) - - // No transitions from INTERNAL_ERROR state. Ignore all. - .addTransition( - VertexState.ERROR, - VertexState.ERROR, - EnumSet.of(VertexEventType.V_INIT, - VertexEventType.V_ROOT_INPUT_FAILED, - VertexEventType.V_SOURCE_VERTEX_STARTED, - VertexEventType.V_START, - VertexEventType.V_ROUTE_EVENT, - VertexEventType.V_TERMINATE, - VertexEventType.V_MANAGER_USER_CODE_ERROR, - VertexEventType.V_TASK_COMPLETED, - VertexEventType.V_TASK_ATTEMPT_COMPLETED, - VertexEventType.V_SOURCE_TASK_ATTEMPT_COMPLETED, - VertexEventType.V_TASK_RESCHEDULED, - VertexEventType.V_INTERNAL_ERROR, - VertexEventType.V_ROOT_INPUT_INITIALIZED, - VertexEventType.V_NULL_EDGE_INITIALIZED, - VertexEventType.V_INPUT_DATA_INFORMATION, - VertexEventType.V_DELETE_SHUFFLE_DATA)) - // create the topology tables - .installTopology(); + VertexEventType.V_SOURCE_TASK_ATTEMPT_COMPLETED, + VertexEventType.V_TASK_RESCHEDULED, + VertexEventType.V_INTERNAL_ERROR, + VertexEventType.V_ROOT_INPUT_INITIALIZED, + VertexEventType.V_NULL_EDGE_INITIALIZED, + VertexEventType.V_INPUT_DATA_INFORMATION, + VertexEventType.V_DELETE_SHUFFLE_DATA)) + // create the topology tables + .installTopology(); private void augmentStateMachine() { stateMachine @@ -681,7 +681,8 @@ private void augmentStateMachine() { .registerStateEnteredCallback(VertexState.RUNNING, STATE_CHANGED_CALLBACK) .registerStateEnteredCallback(VertexState.INITIALIZING, - STATE_CHANGED_CALLBACK);; + STATE_CHANGED_CALLBACK); + ; } private final StateMachineTez stateMachine; @@ -730,7 +731,7 @@ private void augmentStateMachine() { private final String vertexName; private final ProcessorDescriptor processorDescriptor; - + private boolean vertexToBeReconfiguredByManager = false; final AtomicBoolean vmIsInitialized = new AtomicBoolean(false); final AtomicBoolean completelyConfiguredSent = new AtomicBoolean(false); @@ -748,9 +749,9 @@ private void augmentStateMachine() { LinkedHashMap ioIndices = Maps.newLinkedHashMap(); private Map> - rootInputDescriptors; + rootInputDescriptors; private Map> - additionalOutputs; + additionalOutputs; private Map outputCommitters; private Map rootInputSpecs = new HashMap(); private static final InputSpecUpdate DEFAULT_ROOT_INPUT_SPECS = InputSpecUpdate @@ -765,18 +766,17 @@ private void augmentStateMachine() { List pendingTaskEvents = Lists.newLinkedList(); private boolean tasksNotYetScheduled = true; // must be a random access structure - + private final List onDemandRouteEvents = Lists.newArrayListWithCapacity(1000); // Do not send any events if attempt is failed due to INPUT_FAILED_EVENTS. private final Set failedTaskAttemptIDs = Sets.newHashSet(); private final ReadWriteLock onDemandRouteEventsReadWriteLock = new ReentrantReadWriteLock(); private final Lock onDemandRouteEventsReadLock = onDemandRouteEventsReadWriteLock.readLock(); private final Lock onDemandRouteEventsWriteLock = onDemandRouteEventsReadWriteLock.writeLock(); - + List pendingRouteEvents = new LinkedList(); List pendingReportedSrcCompletions = Lists.newLinkedList(); - private RootInputInitializerManager rootInputInitializerManager; VertexManager vertexManager; @@ -813,6 +813,7 @@ static class EventInfo { final Edge eventEdge; final int eventTaskIndex; boolean isObsolete = false; + EventInfo(TezEvent tezEvent, Edge eventEdge, int eventTaskIndex) { this.tezEvent = tezEvent; this.eventEdge = eventEdge; @@ -822,15 +823,14 @@ static class EventInfo { private VertexStatisticsImpl finalStatistics; - - static class IOStatisticsImpl extends org.apache.tez.runtime.api.impl.IOStatistics - implements InputStatistics, OutputStatistics { - + static class IOStatisticsImpl extends org.apache.tez.runtime.api.impl.IOStatistics + implements InputStatistics, OutputStatistics { + @Override public long getDataSize() { return super.getDataSize(); } - + @Override public long getItemsProcessed() { return super.getItemsProcessed(); @@ -978,12 +978,12 @@ public void stopServices() { } public VertexImpl(TezVertexID vertexId, VertexPlan vertexPlan, - String vertexName, Configuration dagConf, EventHandler eventHandler, - TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, Clock clock, - TaskHeartbeatHandler thh, boolean commitVertexOutputs, - AppContext appContext, VertexLocationHint vertexLocationHint, - Map dagVertexGroups, TaskSpecificLaunchCmdOption taskSpecificLaunchCmdOption, - StateChangeNotifier entityStatusTracker, Configuration dagOnlyConf) { + String vertexName, Configuration dagConf, EventHandler eventHandler, + TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, Clock clock, + TaskHeartbeatHandler thh, boolean commitVertexOutputs, + AppContext appContext, VertexLocationHint vertexLocationHint, + Map dagVertexGroups, TaskSpecificLaunchCmdOption taskSpecificLaunchCmdOption, + StateChangeNotifier entityStatusTracker, Configuration dagOnlyConf) { this.vertexId = vertexId; this.vertexPlan = vertexPlan; this.vertexName = StringInterner.intern(vertexName); @@ -1002,7 +1002,7 @@ public VertexImpl(TezVertexID vertexId, VertexPlan vertexPlan, this.clock = clock; this.appContext = appContext; this.commitVertexOutputs = commitVertexOutputs; - this.logIdentifier = this.getVertexId() + " [" + this.getName() + "]"; + this.logIdentifier = this.getVertexId() + " [" + this.getName() + "]"; this.taskCommunicatorManagerInterface = taskCommunicatorManagerInterface; this.taskHeartbeatHandler = thh; @@ -1038,11 +1038,11 @@ public VertexImpl(TezVertexID vertexId, VertexPlan vertexPlan, String javaOptsWithoutLoggerMods = vertexPlan.getTaskConfig().hasJavaOpts() ? vertexPlan.getTaskConfig().getJavaOpts() : null; String logString = vertexConf.get(TezConfiguration.TEZ_TASK_LOG_LEVEL, TezConfiguration.TEZ_TASK_LOG_LEVEL_DEFAULT); - String [] taskLogParams = TezClientUtils.parseLogParams(logString); + String[] taskLogParams = TezClientUtils.parseLogParams(logString); this.javaOpts = TezClientUtils.maybeAddDefaultLoggingJavaOpts(taskLogParams[0], javaOptsWithoutLoggerMods); if (taskSpecificLaunchCmdOpts.hasModifiedLogProperties()) { - String [] taskLogParamsTaskSpecific = taskSpecificLaunchCmdOption.getTaskSpecificLogParams(); + String[] taskLogParamsTaskSpecific = taskSpecificLaunchCmdOption.getTaskSpecificLogParams(); this.javaOptsTaskSpecific = TezClientUtils .maybeAddDefaultLoggingJavaOpts(taskLogParamsTaskSpecific[0], javaOptsWithoutLoggerMods); @@ -1060,7 +1060,6 @@ public VertexImpl(TezVertexID vertexId, VertexPlan vertexPlan, // optionally allow copying this for specific tasks TezClientUtils.addLogParamsToEnv(this.environment, taskLogParams); - this.containerContext = new ContainerContext(this.localResources, appContext.getCurrentDAG().getCredentials(), this.environment, this.javaOpts, this); LOG.info("Default container context for " + logIdentifier + "=" + containerContext + ", Default Resources=" + this.taskResource); @@ -1079,7 +1078,7 @@ public VertexImpl(TezVertexID vertexId, VertexPlan vertexPlan, // Not sending the notifier a parallelism update since this is the initial parallelism this.dagVertexGroups = dagVertexGroups; - + isSpeculationEnabled = vertexConf.getBoolean(TezConfiguration.TEZ_AM_SPECULATION_ENABLED, TezConfiguration.TEZ_AM_SPECULATION_ENABLED_DEFAULT); @@ -1087,7 +1086,7 @@ public VertexImpl(TezVertexID vertexId, VertexPlan vertexPlan, initServices(); maxFailuresPercent = vertexConf.getFloat(TezConfiguration.TEZ_VERTEX_FAILURES_MAXPERCENT, - TezConfiguration.TEZ_VERTEX_FAILURES_MAXPERCENT_DEFAULT); + TezConfiguration.TEZ_VERTEX_FAILURES_MAXPERCENT_DEFAULT); // This "this leak" is okay because the retained pointer is in an // instance variable. @@ -1166,8 +1165,8 @@ public VertexImpl(TezVertexID vertexId, VertexPlan vertexPlan, .append(", TaskCommunicator=").append(taskCommunicatorIdentifier).append(":").append(taskCommName); LOG.info(sb.toString()); cleanupShuffleDataAtVertexLevel = vertexConf.getInt(TezConfiguration.TEZ_AM_VERTEX_CLEANUP_HEIGHT, - TezConfiguration.TEZ_AM_VERTEX_CLEANUP_HEIGHT_DEFAULT) > 0 && - ShuffleUtils.isTezShuffleHandler(vertexConf); + TezConfiguration.TEZ_AM_VERTEX_CLEANUP_HEIGHT_DEFAULT) > 0 && + ShuffleUtils.isTezShuffleHandler(vertexConf); stateMachine = new StateMachineTez( stateMachineFactory.make(this), this); augmentStateMachine(); @@ -1221,7 +1220,7 @@ public VertexPlan getVertexPlan() { public int getDistanceFromRoot() { return distanceFromRoot; } - + @Override public LinkedHashMap getIOIndices() { return ioIndices; @@ -1285,9 +1284,9 @@ public int getSucceededTasks() { public int getRunningTasks() { readLock.lock(); try { - int num=0; + int num = 0; for (Task task : tasks.values()) { - if(task.getState() == TaskState.RUNNING) + if (task.getState() == TaskState.RUNNING) num++; } return num; @@ -1310,7 +1309,6 @@ public TezCounters getAllCounters() { TezCounters counters = new TezCounters(); counters.aggrAllCounters(this.counters); return aggrTaskCounters(counters, tasks.values()); - } finally { readLock.unlock(); } @@ -1323,7 +1321,7 @@ public TezCounters getCachedCounters() { try { // FIXME a better lightweight approach for counters is needed if (fullCounters == null && cachedCounters != null - && ((cachedCountersTimestamp+10000) > System.currentTimeMillis())) { + && ((cachedCountersTimestamp + 10000) > System.currentTimeMillis())) { LOG.info("Asked for counters" + ", cachedCountersTimestamp=" + cachedCountersTimestamp + ", currentTime=" + System.currentTimeMillis()); @@ -1352,7 +1350,7 @@ public void addCounters(final TezCounters tezCounters) { @Override public int getMaxTaskConcurrency() { - return vertexConf.getInt(TezConfiguration.TEZ_AM_VERTEX_MAX_TASK_CONCURRENCY, + return vertexConf.getInt(TezConfiguration.TEZ_AM_VERTEX_MAX_TASK_CONCURRENCY, TezConfiguration.TEZ_AM_VERTEX_MAX_TASK_CONCURRENCY_DEFAULT); } @@ -1367,7 +1365,6 @@ public VertexStats getVertexStats() { VertexStats stats = new VertexStats(); return updateVertexStats(stats, tasks.values()); - } finally { readLock.unlock(); } @@ -1462,7 +1459,6 @@ public static VertexStats updateVertexStats( return stats; } - @Override public List getDiagnostics() { readLock.lock(); @@ -1479,27 +1475,27 @@ public float getProgress() { try { final VertexState state = this.getState(); switch (state) { - case NEW: - case INITED: - case INITIALIZING: - progress = 0.0f; - break; - case RUNNING: - computeProgress(); - break; - case KILLED: - case ERROR: - case FAILED: - case TERMINATING: - progress = 0.0f; - break; - case COMMITTING: - case SUCCEEDED: - progress = 1.0f; - break; - default: - // unknown, do not change progress - break; + case NEW: + case INITED: + case INITIALIZING: + progress = 0.0f; + break; + case RUNNING: + computeProgress(); + break; + case KILLED: + case ERROR: + case FAILED: + case TERMINATING: + progress = 0.0f; + break; + case COMMITTING: + case SUCCEEDED: + progress = 1.0f; + break; + default: + // unknown, do not change progress + break; } return progress; } finally { @@ -1524,7 +1520,7 @@ public float getCompletedTaskProgress() { return 0.0f; } } - return ((float)this.succeededTaskCount/totalTasks); + return ((float) this.succeededTaskCount / totalTasks); } finally { this.readLock.unlock(); } @@ -1643,7 +1639,7 @@ public VertexState getState() { * @return true if setting the value succeeded. */ boolean trySetTerminationCause(VertexTerminationCause trigger) { - if(terminationCause == null){ + if (terminationCause == null) { terminationCause = trigger; return true; } @@ -1651,7 +1647,7 @@ boolean trySetTerminationCause(VertexTerminationCause trigger) { } @Override - public VertexTerminationCause getTerminationCause(){ + public VertexTerminationCause getTerminationCause() { readLock.lock(); try { return terminationCause; @@ -1720,13 +1716,13 @@ public void scheduleSpeculativeTask(TezTaskID taskId) { readLock.unlock(); } } - + void setupEdgeRouting() throws AMUserCodeException { for (Edge e : sourceVertices.values()) { e.routingToBegin(); } } - + private void unsetTasksNotYetScheduled() throws AMUserCodeException { if (tasksNotYetScheduled) { setupEdgeRouting(); @@ -1761,11 +1757,11 @@ private void unsetTasksNotYetScheduled() throws AMUserCodeException { } } } - + TaskSpec createRemoteTaskSpec(int taskIndex) throws AMUserCodeException { return TaskSpec.createBaseTaskSpec(getDAG().getName(), getName(), getTotalTasks(), getProcessorDescriptor(), - getInputSpecList(taskIndex), getOutputSpecList(taskIndex), + getInputSpecList(taskIndex), getOutputSpecList(taskIndex), getGroupInputSpecList(), vertexOnlyConf); } @@ -1815,34 +1811,34 @@ public void scheduleTasks(List tasksToSchedule) { throw new TezUncheckedException(e); } } - + @Override public void reconfigureVertex(int parallelism, - @Nullable VertexLocationHint locationHint, - @Nullable Map sourceEdgeProperties) throws AMUserCodeException { + @Nullable VertexLocationHint locationHint, + @Nullable Map sourceEdgeProperties) throws AMUserCodeException { setParallelismWrapper(parallelism, locationHint, sourceEdgeProperties, null, true); } - + @Override public void reconfigureVertex(@Nullable Map rootInputSpecUpdate, - int parallelism, - @Nullable VertexLocationHint locationHint) throws AMUserCodeException { + int parallelism, + @Nullable VertexLocationHint locationHint) throws AMUserCodeException { setParallelism(parallelism, locationHint, null, rootInputSpecUpdate, true); } @Override public void reconfigureVertex(int parallelism, - @Nullable VertexLocationHint locationHint, - @Nullable Map sourceEdgeProperties, - @Nullable Map rootInputSpecUpdate) throws AMUserCodeException { + @Nullable VertexLocationHint locationHint, + @Nullable Map sourceEdgeProperties, + @Nullable Map rootInputSpecUpdate) throws AMUserCodeException { setParallelismWrapper(parallelism, locationHint, sourceEdgeProperties, rootInputSpecUpdate, true); } - + @Override public void setParallelism(int parallelism, VertexLocationHint vertexLocationHint, - Map sourceEdgeManagers, - Map rootInputSpecUpdates, boolean fromVertexManager) - throws AMUserCodeException { + Map sourceEdgeManagers, + Map rootInputSpecUpdates, boolean fromVertexManager) + throws AMUserCodeException { // temporarily support conversion of edge manager to edge property Map sourceEdgeProperties = Maps.newHashMap(); readLock.lock(); @@ -1868,9 +1864,9 @@ public void setParallelism(int parallelism, VertexLocationHint vertexLocationHin } private void setParallelismWrapper(int parallelism, VertexLocationHint vertexLocationHint, - Map sourceEdgeProperties, - Map rootInputSpecUpdates, - boolean fromVertexManager) throws AMUserCodeException { + Map sourceEdgeProperties, + Map rootInputSpecUpdates, + boolean fromVertexManager) throws AMUserCodeException { Preconditions.checkArgument(parallelism >= 0, "Parallelism must be >=0. Value: " + parallelism + " for vertex: " + logIdentifier); writeLock.lock(); @@ -1896,7 +1892,7 @@ private void setParallelismWrapper(int parallelism, VertexLocationHint vertexLoc + " context.vertexReconfigurationPlanned() before re-configuring the vertex." + " vertexId=" + logIdentifier); } - + // Input initializer/Vertex Manager/1-1 split expected to set parallelism. if (numTasks == -1) { if (getState() != VertexState.INITIALIZING) { @@ -1905,8 +1901,8 @@ private void setParallelismWrapper(int parallelism, VertexLocationHint vertexLoc + " for vertex: " + logIdentifier); } - if(sourceEdgeProperties != null) { - for(Map.Entry entry : sourceEdgeProperties.entrySet()) { + if (sourceEdgeProperties != null) { + for (Map.Entry entry : sourceEdgeProperties.entrySet()) { LOG.info("Replacing edge manager for source:" + entry.getKey() + " destination: " + getLogIdentifier()); Vertex sourceVertex = appContext.getCurrentDAG().getVertex(entry.getKey()); @@ -1929,7 +1925,7 @@ private void setParallelismWrapper(int parallelism, VertexLocationHint vertexLoc .checkState( rootInputSpecUpdateEntry.getValue().isForAllWorkUnits() || (rootInputSpecUpdateEntry.getValue().getAllNumPhysicalInputs() != null && rootInputSpecUpdateEntry - .getValue().getAllNumPhysicalInputs().size() == parallelism), + .getValue().getAllNumPhysicalInputs().size() == parallelism), "Not enough input spec updates for root input named " + rootInputSpecUpdateEntry.getKey()); } @@ -1954,9 +1950,9 @@ private void setParallelismWrapper(int parallelism, VertexLocationHint vertexLoc // for a vertex to start. Preconditions.checkState(rootInputSpecUpdates == null, "Root Input specs can only be updated when the vertex is configured with -1 tasks"); - + int oldNumTasks = numTasks; - + // start buffering incoming events so that we can re-route existing events for (Edge edge : sourceVertices.values()) { edge.startEventBuffering(); @@ -1979,20 +1975,20 @@ private void setParallelismWrapper(int parallelism, VertexLocationHint vertexLoc } } Preconditions.checkState(this.numTasks == parallelism, getLogIdentifier()); - + // set new vertex location hints setVertexLocationHint(vertexLocationHint); LOG.info("Vertex " + getLogIdentifier() + " parallelism set to " + parallelism + " from " + oldNumTasks); - + // notify listeners stateChangeNotifier.stateChanged(vertexId, new VertexStateUpdateParallelismUpdated(vertexName, numTasks, oldNumTasks)); assert tasks.size() == numTasks; // set new edge managers - if(sourceEdgeProperties != null) { - for(Map.Entry entry : sourceEdgeProperties.entrySet()) { + if (sourceEdgeProperties != null) { + for (Map.Entry entry : sourceEdgeProperties.entrySet()) { LOG.info("Replacing edge manager for source:" + entry.getKey() + " destination: " + getLogIdentifier()); Vertex sourceVertex = appContext.getCurrentDAG().getVertex(entry.getKey()); @@ -2010,7 +2006,6 @@ private void setParallelismWrapper(int parallelism, VertexLocationHint vertexLoc edge.stopEventBuffering(); } } - } finally { writeLock.unlock(); } @@ -2028,7 +2023,7 @@ public void setVertexLocationHint(VertexLocationHint vertexLocationHint) { writeLock.unlock(); } } - + @Override public void vertexReconfigurationPlanned() { writeLock.lock(); @@ -2040,7 +2035,7 @@ public void vertexReconfigurationPlanned() { this.vertexToBeReconfiguredByManager = true; } finally { writeLock.unlock(); - } + } } @Override @@ -2056,10 +2051,9 @@ public void doneReconfiguringVertex() { Preconditions.checkState(getInternalState() == VertexState.INITIALIZING, "Vertex: " + getLogIdentifier()); } - } finally { writeLock.unlock(); - } + } } @Override @@ -2076,7 +2070,7 @@ public void handle(VertexEvent event) { writeLock.lock(); VertexState oldState = getInternalState(); try { - getStateMachine().doTransition(event.getType(), event); + getStateMachine().doTransition(event.getType(), event); } catch (InvalidStateTransitonException e) { String message = "Invalid event " + event.getType() + " on vertex " + this.vertexName + @@ -2104,9 +2098,7 @@ public void handle(VertexEvent event) { + getInternalState() + " due to event " + event.getType()); } - } - - finally { + } finally { writeLock.unlock(); } } @@ -2114,7 +2106,7 @@ public void handle(VertexEvent event) { private VertexState getInternalState() { readLock.lock(); try { - return getStateMachine().getCurrentState(); + return getStateMachine().getCurrentState(); } finally { readLock.unlock(); } @@ -2146,7 +2138,7 @@ void logJobHistoryVertexInitializedEvent() { getProcessorName(), getAdditionalInputs(), initGeneratedEvents, servicePluginInfo); this.appContext.getHistoryHandler().handle( - new DAGHistoryEvent(getDAGId(), initEvt)); + new DAGHistoryEvent(getDAGId(), initEvt)); } } @@ -2169,7 +2161,7 @@ void logVertexConfigurationDoneEvent() { VertexConfigurationDoneEvent reconfigureDoneEvent = new VertexConfigurationDoneEvent(vertexId, clock.getTime(), numTasks, taskLocationHints == null ? null : VertexLocationHint.create(Lists.newArrayList(taskLocationHints)), - sourceEdgeProperties, rootInputSpecs, setParallelismCalledFlag); + sourceEdgeProperties, rootInputSpecs, setParallelismCalledFlag); this.appContext.getHistoryHandler().handle( new DAGHistoryEvent(getDAGId(), reconfigureDoneEvent)); } @@ -2274,7 +2266,7 @@ public Void run() throws Exception { return null; } }; - ListenableFuture commitFuture = + ListenableFuture commitFuture = vertex.getAppContext().getExecService().submit(commitCallableEvent); Futures.addCallback(commitFuture, commitCallableEvent.getCallback(), GuavaShim.directExecutor()); vertex.commitFutures.put(outputName, commitFuture); @@ -2319,8 +2311,8 @@ static VertexState checkTasksForCompletion(final VertexImpl vertex) { boolean vertexFailuresBelowThreshold = (vertex.succeededTaskCount + vertex.failedTaskCount == vertex.numTasks) && (vertex.failedTaskCount * 100 <= vertex.maxFailuresPercent * vertex.numTasks); - if((vertexSucceeded || vertexFailuresBelowThreshold) && vertex.terminationCause == null) { - if(vertexSucceeded) { + if ((vertexSucceeded || vertexFailuresBelowThreshold) && vertex.terminationCause == null) { + if (vertexSucceeded) { LOG.info("All tasks have succeeded, vertex:" + vertex.logIdentifier); if (vertex.cleanupShuffleDataAtVertexLevel) { @@ -2383,12 +2375,12 @@ static VertexState checkTasksForCompletion(final VertexImpl vertex) { //triggered by commit_complete static VertexState checkCommitsForCompletion(final VertexImpl vertex) { LOG.info("Commits completion: " - + constructCheckTasksForCompletionLog(vertex)); + + constructCheckTasksForCompletionLog(vertex)); // terminationCause is null mean commit is succeeded, otherwise terminationCause will be set. if (vertex.terminationCause == null) { Preconditions.checkState(vertex.getState() == VertexState.COMMITTING, "Vertex should be in COMMITTING state, but in " + vertex.getState() - + ", vertex:" + vertex.getLogIdentifier()); + + ", vertex:" + vertex.getLogIdentifier()); if (vertex.commitFutures.isEmpty()) { // move from COMMITTING to SUCCEEDED return vertex.finished(VertexState.SUCCEEDED); @@ -2421,14 +2413,14 @@ private static VertexState finishWithTerminationCause(VertexImpl vertex) { * The task-kill messages are only sent once. */ void tryEnactKill(VertexTerminationCause trigger, - TaskTerminationCause taskterminationCause) { + TaskTerminationCause taskterminationCause) { // In most cases the dag is shutting down due to some error TaskAttemptTerminationCause errCause = TaskAttemptTerminationCause.TERMINATED_AT_SHUTDOWN; if (taskterminationCause == TaskTerminationCause.DAG_KILL) { errCause = TaskAttemptTerminationCause.TERMINATED_BY_CLIENT; } - if(trySetTerminationCause(trigger)){ - String msg = "Killing tasks in vertex: " + logIdentifier + " due to trigger: " + trigger; + if (trySetTerminationCause(trigger)) { + String msg = "Killing tasks in vertex: " + logIdentifier + " due to trigger: " + trigger; LOG.info(msg); for (Task task : tasks.values()) { eventHandler.handle( // attempt was terminated because the vertex is shutting down @@ -2438,7 +2430,7 @@ void tryEnactKill(VertexTerminationCause trigger, } VertexState finished(VertexState finalState, - VertexTerminationCause termCause, String diag) { + VertexTerminationCause termCause, String diag) { if (finishTime == 0) setFinishTime(); if (termCause != null) { trySetTerminationCause(termCause); @@ -2533,13 +2525,12 @@ VertexState finished(VertexState finalState) { return finished(finalState, null, null); } - private void initializeCommitters() throws Exception { if (!this.additionalOutputSpecs.isEmpty()) { LOG.info("Setting up committers for vertex " + logIdentifier + ", numAdditionalOutputs=" + additionalOutputs.size()); - for (Entry> entry: - additionalOutputs.entrySet()) { + for (Entry> entry : + additionalOutputs.entrySet()) { final String outputName = entry.getKey(); final RootInputLeafOutput od = entry.getValue(); if (od.getControllerDescriptor() == null @@ -2658,24 +2649,24 @@ private TaskImpl createTask(int taskIndex) { this.taskHeartbeatHandler, this.appContext, (this.targetVertices != null ? - this.targetVertices.isEmpty() : true), + this.targetVertices.isEmpty() : true), this.taskResource, conContext, this.stateChangeNotifier, this); } - + private void createTasks() { - for (int i=0; i < this.numTasks; ++i) { + for (int i = 0; i < this.numTasks; ++i) { TaskImpl task = createTask(i); this.addTask(task); - if(LOG.isDebugEnabled()) { + if (LOG.isDebugEnabled()) { LOG.debug("Created task for vertex " + logIdentifier + ": " + task.getTaskID()); } } } - + private void addTasks(int newNumTasks) { Preconditions.checkArgument(newNumTasks > this.numTasks, getLogIdentifier()); int initialNumTasks = this.numTasks; @@ -2683,13 +2674,13 @@ private void addTasks(int newNumTasks) { TaskImpl task = createTask(i); this.addTask(task); this.numTasks++; - if(LOG.isDebugEnabled()) { + if (LOG.isDebugEnabled()) { LOG.debug("Created task for vertex " + logIdentifier + ": " + task.getTaskID()); } } } - + private void removeTasks(int newNumTasks) { Preconditions.checkArgument(newNumTasks < this.numTasks, getLogIdentifier()); // assign to local variable of LinkedHashMap to make sure that changing @@ -2718,7 +2709,6 @@ private void removeTasks(int newNumTasks) { } } - private VertexState setupVertex() { this.initTimeRequested = clock.getTime(); @@ -2808,7 +2798,7 @@ private VertexState setupVertex() { pendingVmEvents.clear(); } } catch (AMUserCodeException e) { - String msg = "Exception in " + e.getSource()+ ", vertex:" + logIdentifier; + String msg = "Exception in " + e.getSource() + ", vertex:" + logIdentifier; LOG.error(msg, e); finished(VertexState.FAILED, VertexTerminationCause.AM_USERCODE_FAILURE, msg + ", " + e.getMessage() + ", " + ExceptionUtils.getStackTrace(e.getCause())); @@ -2819,7 +2809,7 @@ private VertexState setupVertex() { private boolean isVertexInitSkippedInParentVertices() { for (Map.Entry entry : sourceVertices.entrySet()) { - if(!(((VertexImpl) entry.getKey()).isVertexInitSkipped())) { + if (!(((VertexImpl) entry.getKey()).isVertexInitSkipped())) { return false; } } @@ -2861,7 +2851,7 @@ && isVertexInitSkippedInParentVertices()) { } this.vertexManager = new VertexManager( VertexManagerPluginDescriptor.create(NoOpVertexManager.class.getName()) - .setUserPayload(UserPayload.create(ByteBuffer.wrap(out.toByteArray()))), + .setUserPayload(UserPayload.create(ByteBuffer.wrap(out.toByteArray()))), dagUgi, this, appContext, stateChangeNotifier); isVertexInitSkipped = true; return; @@ -2872,21 +2862,21 @@ && isVertexInitSkippedInParentVertices()) { boolean hasCustom = false; if (sourceVertices != null) { for (Edge edge : sourceVertices.values()) { - switch(edge.getEdgeProperty().getDataMovementType()) { - case SCATTER_GATHER: - hasBipartite = true; - break; - case ONE_TO_ONE: - hasOneToOne = true; - break; - case BROADCAST: - break; - case CUSTOM: - hasCustom = true; - break; - default: - throw new TezUncheckedException("Unknown data movement type: " + - edge.getEdgeProperty().getDataMovementType()); + switch (edge.getEdgeProperty().getDataMovementType()) { + case SCATTER_GATHER: + hasBipartite = true; + break; + case ONE_TO_ONE: + hasOneToOne = true; + break; + case BROADCAST: + break; + case CUSTOM: + hasCustom = true; + break; + default: + throw new TezUncheckedException("Unknown data movement type: " + + edge.getEdgeProperty().getDataMovementType()); } } } @@ -2935,9 +2925,9 @@ && isVertexInitSkippedInParentVertices()) { } } } - - private static List getTaskAttemptIdentifiers(DAG dag, - List taIds) { + + private static List getTaskAttemptIdentifiers(DAG dag, + List taIds) { List attempts = new ArrayList(taIds.size()); String dagName = dag.getName(); for (TezTaskAttemptID taId : taIds) { @@ -2946,9 +2936,9 @@ private static List getTaskAttemptIdentifiers(DAG dag, } return attempts; } - - private static TaskAttemptIdentifier getTaskAttemptIdentifier(String dagName, String vertexName, - TezTaskAttemptID taId) { + + private static TaskAttemptIdentifier getTaskAttemptIdentifier(String dagName, String vertexName, + TezTaskAttemptID taId) { return new TaskAttemptIdentifierImpl(dagName, vertexName, taId); } @@ -2967,12 +2957,12 @@ public VertexState transition(VertexImpl vertex, VertexEvent vertexEvent) { + vertex.logIdentifier); Preconditions.checkState( (vertex.sourceVertices == null || vertex.sourceVertices.containsKey(otherVertex) || - vertex.targetVertices == null || vertex.targetVertices.containsKey(otherVertex)), + vertex.targetVertices == null || vertex.targetVertices.containsKey(otherVertex)), "Not connected to vertex " + otherVertex.getLogIdentifier() + " from vertex: " + vertex.logIdentifier); LOG.info("Edge initialized for connection to vertex " + otherVertex.getLogIdentifier() + " at vertex : " + vertex.logIdentifier); vertex.uninitializedEdges.remove(edge); - if(vertex.getState() == VertexState.INITIALIZING && vertex.canInitVertex()) { + if (vertex.getState() == VertexState.INITIALIZING && vertex.canInitVertex()) { // Vertex in Initialing state and can init. Do init. return VertexInitializedTransition.doTransition(vertex); } @@ -2990,22 +2980,22 @@ public VertexState transition(VertexImpl vertex, VertexEvent vertexEvent) { // with desired state, for the cases that DAG is completed VertexState desiredState = recoverEvent.getDesiredState(); switch (desiredState) { - case SUCCEEDED: - vertex.succeededTaskCount = vertex.numTasks; - vertex.completedTaskCount = vertex.numTasks; - break; - case KILLED: - vertex.killedTaskCount = vertex.numTasks; - break; - case FAILED: - case ERROR: - vertex.failedTaskCount = vertex.numTasks; - break; - default: - LOG.info("Unhandled desired state provided by DAG" - + ", vertex=" + vertex.logIdentifier - + ", state=" + desiredState); - return vertex.finished(VertexState.ERROR); + case SUCCEEDED: + vertex.succeededTaskCount = vertex.numTasks; + vertex.completedTaskCount = vertex.numTasks; + break; + case KILLED: + vertex.killedTaskCount = vertex.numTasks; + break; + case FAILED: + case ERROR: + vertex.failedTaskCount = vertex.numTasks; + break; + default: + LOG.info("Unhandled desired state provided by DAG" + + ", vertex=" + vertex.logIdentifier + + ", state=" + desiredState); + return vertex.finished(VertexState.ERROR); } LOG.info("DAG informed vertices of its final completed state" @@ -3014,7 +3004,7 @@ public VertexState transition(VertexImpl vertex, VertexEvent vertexEvent) { return vertex.finished(recoverEvent.getDesiredState()); } } - + public static class InitTransition implements MultipleArcTransition { @@ -3114,8 +3104,8 @@ private VertexState handleInitEvent(VertexImpl vertex) { return VertexState.INITIALIZING; } throw new TezUncheckedException(vertex.getLogIdentifier() + - " has -1 tasks but does not have input initializers, " + - "1-1 uninited sources or custom vertex manager to set it at runtime"); + " has -1 tasks but does not have input initializers, " + + "1-1 uninited sources or custom vertex manager to set it at runtime"); } } else { LOG.info("Creating " + vertex.numTasks + " tasks for vertex: " + vertex.logIdentifier); @@ -3173,7 +3163,7 @@ void startIfPossible() { // Trigger a start event to ensure route events are seen before // a start event. LOG.info("Triggering start event for vertex: " + logIdentifier + - " with distanceFromRoot: " + distanceFromRoot ); + " with distanceFromRoot: " + distanceFromRoot); eventHandler.handle(new VertexEvent(vertexId, VertexEventType.V_START)); } @@ -3210,15 +3200,15 @@ public VertexState transition(VertexImpl vertex, VertexEvent event) { if (state == VertexState.INITIALIZING) { try { vertex.vertexManager.onRootVertexInitialized(liInitEvent.getInputName(), vertex - .getAdditionalInputs().get(liInitEvent.getInputName()).getIODescriptor(), + .getAdditionalInputs().get(liInitEvent.getInputName()).getIODescriptor(), liInitEvent.getEvents()); } catch (AMUserCodeException e) { - String msg = "Exception in " + e.getSource() + ", vertex:" + vertex.getLogIdentifier(); - LOG.error(msg, e); - vertex.finished(VertexState.FAILED, - VertexTerminationCause.AM_USERCODE_FAILURE, msg - + "," + ExceptionUtils.getStackTrace(e.getCause())); - return VertexState.FAILED; + String msg = "Exception in " + e.getSource() + ", vertex:" + vertex.getLogIdentifier(); + LOG.error(msg, e); + vertex.finished(VertexState.FAILED, + VertexTerminationCause.AM_USERCODE_FAILURE, msg + + "," + ExceptionUtils.getStackTrace(e.getCause())); + return VertexState.FAILED; } } @@ -3228,7 +3218,7 @@ public VertexState transition(VertexImpl vertex, VertexEvent event) { vertex.rootInputInitializerManager.shutdown(); vertex.rootInputInitializerManager = null; } - + // the return of these events from the VM will complete initialization and move into // INITED state if possible via InputDataInformationTransition @@ -3283,9 +3273,9 @@ public static class SourceVertexStartedTransition implements @Override public void transition(VertexImpl vertex, VertexEvent event) { VertexEventSourceVertexStarted startEvent = - (VertexEventSourceVertexStarted) event; + (VertexEventSourceVertexStarted) event; int distanceFromRoot = startEvent.getSourceDistanceFromRoot() + 1; - if(vertex.distanceFromRoot < distanceFromRoot) { + if (vertex.distanceFromRoot < distanceFromRoot) { vertex.distanceFromRoot = distanceFromRoot; } vertex.numStartedSourceVertices++; @@ -3338,7 +3328,7 @@ boolean canInitVertex() { } public static class StartWhileInitializingTransition implements - SingleArcTransition { + SingleArcTransition { @Override public void transition(VertexImpl vertex, VertexEvent event) { @@ -3351,11 +3341,10 @@ public void transition(VertexImpl vertex, VertexEvent event) { vertex.startSignalPending = true; vertex.startTimeRequested = vertex.clock.getTime(); } - } - + public static class StartTransition implements - MultipleArcTransition { + MultipleArcTransition { @Override public VertexState transition(VertexImpl vertex, VertexEvent event) { @@ -3368,7 +3357,7 @@ public VertexState transition(VertexImpl vertex, VertexEvent event) { return vertex.startVertex(); } } - + private void maybeSendConfiguredEvent() { // the vertex is fully configured by the time it starts. Always notify completely configured // unless the vertex manager has told us that it is going to reconfigure it further @@ -3397,7 +3386,7 @@ private VertexState startVertex() { try { vertexManager.onVertexStarted(getTaskAttemptIdentifiers(dag, pendingReportedSrcCompletions)); } catch (AMUserCodeException e) { - String msg = "Exception in " + e.getSource() +", vertex=" + logIdentifier; + String msg = "Exception in " + e.getSource() + ", vertex=" + logIdentifier; LOG.error(msg, e); addDiagnostic(msg + "," + ExceptionUtils.getStackTrace(e.getCause())); tryEnactKill(VertexTerminationCause.AM_USERCODE_FAILURE, TaskTerminationCause.AM_USERCODE_FAILURE); @@ -3405,7 +3394,7 @@ private VertexState startVertex() { } pendingReportedSrcCompletions.clear(); logJobHistoryVertexStartedEvent(); - + // the vertex is fully configured by the time it starts. Always notify completely configured // unless the vertex manager has told us that it is going to reconfigure it further. // If the vertex was pre-configured then the event would have been sent out earlier. Calling again @@ -3414,7 +3403,7 @@ private VertexState startVertex() { // or vertex added (future). Simplify these cases by sending the event now automatically for the // user as if they had invoked the planned()/done() API's. maybeSendConfiguredEvent(); - + // TODO: Metrics //job.metrics.runningJob(job); @@ -3432,9 +3421,9 @@ private VertexState startVertex() { // If we have no tasks, just transition to vertex completed if (this.numTasks == 0) { eventHandler.handle(new VertexEvent( - this.vertexId, VertexEventType.V_COMPLETED)); + this.vertexId, VertexEventType.V_COMPLETED)); } - + return VertexState.RUNNING; } @@ -3522,14 +3511,14 @@ public VertexState transition(VertexImpl vertex, VertexEvent event) { LOG.error(msg, fe.getError()); if (vertex.getState() == VertexState.RUNNING) { vertex.addDiagnostic(msg - + ", " + ExceptionUtils.getStackTrace(fe.getError().getCause())); + + ", " + ExceptionUtils.getStackTrace(fe.getError().getCause())); vertex.tryEnactKill(VertexTerminationCause.ROOT_INPUT_INIT_FAILURE, TaskTerminationCause.AM_USERCODE_FAILURE); return VertexState.TERMINATING; } else { vertex.finished(VertexState.FAILED, VertexTerminationCause.ROOT_INPUT_INIT_FAILURE, msg - + ", " + ExceptionUtils.getStackTrace(fe.getError().getCause())); + + ", " + ExceptionUtils.getStackTrace(fe.getError().getCause())); return VertexState.FAILED; } } @@ -3538,7 +3527,7 @@ public VertexState transition(VertexImpl vertex, VertexEvent event) { // Task-start has been moved out of InitTransition, so this arc simply // hardcodes 0 for both map and reduce finished tasks. private static class TerminateNewVertexTransition - implements SingleArcTransition { + implements SingleArcTransition { @Override public void transition(VertexImpl vertex, VertexEvent event) { VertexEventTermination vet = (VertexEventTermination) event; @@ -3550,7 +3539,7 @@ public void transition(VertexImpl vertex, VertexEvent event) { } private static class TerminateInitedVertexTransition - implements SingleArcTransition { + implements SingleArcTransition { @Override public void transition(VertexImpl vertex, VertexEvent event) { VertexEventTermination vet = (VertexEventTermination) event; @@ -3574,9 +3563,13 @@ public void transition(VertexImpl vertex, VertexEvent event) { vertex.addDiagnostic("Vertex received Kill while in RUNNING state."); VertexEventTermination vet = (VertexEventTermination) event; VertexTerminationCause trigger = vet.getTerminationCause(); - switch(trigger){ - case DAG_TERMINATED: vertex.tryEnactKill(trigger, TaskTerminationCause.DAG_KILL); break; - case OWN_TASK_FAILURE: vertex.tryEnactKill(trigger, TaskTerminationCause.OTHER_TASK_FAILURE); break; + switch (trigger) { + case DAG_TERMINATED: + vertex.tryEnactKill(trigger, TaskTerminationCause.DAG_KILL); + break; + case OWN_TASK_FAILURE: + vertex.tryEnactKill(trigger, TaskTerminationCause.OTHER_TASK_FAILURE); + break; case ROOT_INPUT_INIT_FAILURE: case COMMIT_FAILURE: case INVALID_NUM_OF_TASKS: @@ -3585,7 +3578,9 @@ public void transition(VertexImpl vertex, VertexEvent event) { case AM_USERCODE_FAILURE: case VERTEX_RERUN_IN_COMMITTING: case VERTEX_RERUN_AFTER_COMMIT: - case OTHER_VERTEX_FAILURE: vertex.tryEnactKill(trigger, TaskTerminationCause.OTHER_VERTEX_FAILURE); break; + case OTHER_VERTEX_FAILURE: + vertex.tryEnactKill(trigger, TaskTerminationCause.OTHER_VERTEX_FAILURE); + break; default://should not occur throw new TezUncheckedException("VertexKilledTransition: event.terminationCause is unexpected: " + trigger); } @@ -3596,14 +3591,14 @@ public void transition(VertexImpl vertex, VertexEvent event) { } private static class VertexKilledWhileCommittingTransition - implements SingleArcTransition { + implements SingleArcTransition { @Override public void transition(VertexImpl vertex, VertexEvent event) { VertexEventTermination vet = (VertexEventTermination) event; VertexTerminationCause trigger = vet.getTerminationCause(); String msg = "Vertex received Kill while in COMMITTING state, terminationCause=" - + trigger +", vertex=" + vertex.logIdentifier; + + trigger + ", vertex=" + vertex.logIdentifier; LOG.info(msg); vertex.addDiagnostic(msg); vertex.trySetTerminationCause(trigger); @@ -3612,14 +3607,14 @@ public void transition(VertexImpl vertex, VertexEvent event) { } private static class VertexManagerUserCodeErrorTransition implements - MultipleArcTransition { + MultipleArcTransition { @Override public VertexState transition(VertexImpl vertex, VertexEvent event) { VertexEventManagerUserCodeError errEvent = ((VertexEventManagerUserCodeError) event); AMUserCodeException e = errEvent.getError(); String msg = "Exception in " + e.getSource() + ", vertex:" + vertex.getLogIdentifier(); LOG.error(msg, e); - + if (vertex.getState() == VertexState.RUNNING || vertex.getState() == VertexState.COMMITTING) { vertex.addDiagnostic(msg + "," + ExceptionUtils.getStackTrace(e.getCause())); vertex.tryEnactKill(VertexTerminationCause.AM_USERCODE_FAILURE, @@ -3629,27 +3624,26 @@ public VertexState transition(VertexImpl vertex, VertexEvent event) { } else { vertex.finished(VertexState.FAILED, VertexTerminationCause.AM_USERCODE_FAILURE, msg - + ", " + ExceptionUtils.getStackTrace(e.getCause())); + + ", " + ExceptionUtils.getStackTrace(e.getCause())); return VertexState.FAILED; } } } - + /** * Here, the Vertex is being told that one of it's source task-attempts * completed. */ private static class SourceTaskAttemptCompletedEventTransition implements - MultipleArcTransition { + MultipleArcTransition { @Override public VertexState transition(VertexImpl vertex, VertexEvent event) { VertexEventTaskAttemptCompleted completionEvent = ((VertexEventSourceTaskAttemptCompleted) event).getCompletionEvent(); LOG.info("Source task attempt completed for vertex: " + vertex.getLogIdentifier() - + " attempt: " + completionEvent.getTaskAttemptId() - + " with state: " + completionEvent.getTaskAttemptState() - + " vertexState: " + vertex.getState()); - + + " attempt: " + completionEvent.getTaskAttemptId() + + " with state: " + completionEvent.getTaskAttemptState() + + " vertexState: " + vertex.getState()); if (TaskAttemptStateInternal.SUCCEEDED.equals(completionEvent .getTaskAttemptState())) { @@ -3660,9 +3654,9 @@ public VertexState transition(VertexImpl vertex, VertexEvent event) { // Inform the vertex manager about the source task completing. TezTaskAttemptID taId = completionEvent.getTaskAttemptId(); vertex.vertexManager.onSourceTaskCompleted( - getTaskAttemptIdentifier(vertex.dag.getName(), - vertex.dag.getVertex(taId.getVertexID()).getName(), - taId)); + getTaskAttemptIdentifier(vertex.dag.getName(), + vertex.dag.getVertex(taId.getVertexID()).getName(), + taId)); } catch (AMUserCodeException e) { String msg = "Exception in " + e.getSource() + ", vertex:" + vertex.getLogIdentifier(); LOG.error(msg, e); @@ -3684,7 +3678,7 @@ private static class TaskAttemptCompletedEventTransition implements @Override public void transition(VertexImpl vertex, VertexEvent event) { VertexEventTaskAttemptCompleted completionEvent = - ((VertexEventTaskAttemptCompleted) event); + ((VertexEventTaskAttemptCompleted) event); // If different tasks were connected to different destination vertices // then this would need to be sent via the edges @@ -3694,7 +3688,7 @@ public void transition(VertexImpl vertex, VertexEvent event) { vertex.eventHandler.handle( new VertexEventSourceTaskAttemptCompleted( targetVertex.getVertexId(), completionEvent) - ); + ); } } } @@ -3722,7 +3716,7 @@ public VertexState transition(VertexImpl vertex, VertexEvent event) { taskFailed(vertex, task); if (vertex.failedTaskCount * 100 > vertex.maxFailuresPercent * vertex.numTasks) { LOG.info("Failing vertex: " + vertex.logIdentifier + - " because task failed: " + taskEvent.getTaskID()); + " because task failed: " + taskEvent.getTaskID()); vertex.tryEnactKill(VertexTerminationCause.OWN_TASK_FAILURE, TaskTerminationCause.OTHER_TASK_FAILURE); forceTransitionToKillWait = true; } @@ -3731,7 +3725,7 @@ public VertexState transition(VertexImpl vertex, VertexEvent event) { } VertexState state = VertexImpl.checkTasksForCompletion(vertex); - if(state == VertexState.RUNNING && forceTransitionToKillWait){ + if (state == VertexState.RUNNING && forceTransitionToKillWait) { return VertexState.TERMINATING; } @@ -3747,8 +3741,8 @@ private void taskSucceeded(VertexImpl vertex, Task task) { private void taskFailed(VertexImpl vertex, Task task) { vertex.failedTaskCount++; vertex.addDiagnostic("Task failed" - + ", taskId=" + task.getTaskID() - + ", diagnostics=" + task.getDiagnostics()); + + ", taskId=" + task.getTaskID() + + ", diagnostics=" + task.getDiagnostics()); // TODO Metrics //vertex.metrics.failedTask(task); } @@ -3781,7 +3775,7 @@ public VertexState transition(VertexImpl vertex, VertexEvent event) { } private static class VertexShuffleDeleteTransition implements - SingleArcTransition { + SingleArcTransition { @Override public void transition(VertexImpl vertex, VertexEvent event) { @@ -3791,7 +3785,7 @@ public void transition(VertexImpl vertex, VertexEvent event) { // check if all the child vertices are completed if (incompleteChildrenVertices == 0) { LOG.info("Vertex shuffle data deletion for vertex name: " + - vertex.getName() + " with vertex id: " + vertex.getVertexId()); + vertex.getName() + " with vertex id: " + vertex.getVertexId()); // Get nodes of all the task attempts in vertex Set nodes = Sets.newHashSet(); Map tasksMap = vertex.getTasks(); @@ -3802,7 +3796,7 @@ public void transition(VertexImpl vertex, VertexEvent event) { }); }); vertex.appContext.getAppMaster().vertexComplete( - vertex.vertexId, nodes); + vertex.vertexId, nodes); } else { LOG.debug("The number of incomplete child vertex are {} for the vertex {}", incompleteChildrenVertices, vertex.vertexId); @@ -3811,7 +3805,7 @@ public void transition(VertexImpl vertex, VertexEvent event) { } private static class TaskCompletedAfterVertexSuccessTransition implements - MultipleArcTransition { + MultipleArcTransition { @Override public VertexState transition(VertexImpl vertex, VertexEvent event) { VertexEventTaskCompleted vEvent = (VertexEventTaskCompleted) event; @@ -3819,8 +3813,8 @@ public VertexState transition(VertexImpl vertex, VertexEvent event) { String diagnosticMsg; if (vEvent.getState() == TaskState.FAILED) { finalState = VertexState.FAILED; - diagnosticMsg = "Vertex " + vertex.logIdentifier +" failed as task " + vEvent.getTaskID() + - " failed after vertex succeeded."; + diagnosticMsg = "Vertex " + vertex.logIdentifier + " failed as task " + vEvent.getTaskID() + + " failed after vertex succeeded."; } else { finalState = VertexState.ERROR; diagnosticMsg = "Vertex " + vertex.logIdentifier + " error as task " + vEvent.getTaskID() + @@ -3832,14 +3826,14 @@ public VertexState transition(VertexImpl vertex, VertexEvent event) { } } - private static class TaskRescheduledWhileCommittingTransition implements - SingleArcTransition { + private static class TaskRescheduledWhileCommittingTransition implements + SingleArcTransition { @Override public void transition(VertexImpl vertex, VertexEvent event) { // terminate any running tasks String diagnosticMsg = vertex.getLogIdentifier() + " failed due to in-committing rescheduling of " - + ((VertexEventTaskReschedule)event).getTaskID(); + + ((VertexEventTaskReschedule) event).getTaskID(); LOG.info(diagnosticMsg); vertex.addDiagnostic(diagnosticMsg); vertex.tryEnactKill(VertexTerminationCause.VERTEX_RERUN_IN_COMMITTING, @@ -3849,7 +3843,7 @@ public void transition(VertexImpl vertex, VertexEvent event) { } private static class TaskRescheduledAfterVertexSuccessTransition implements - MultipleArcTransition { + MultipleArcTransition { @Override public VertexState transition(VertexImpl vertex, VertexEvent event) { @@ -3857,7 +3851,7 @@ public VertexState transition(VertexImpl vertex, VertexEvent event) { || vertex.outputCommitters.isEmpty() // no committer || !vertex.commitVertexOutputs) { // committer does not commit on vertex success LOG.info(vertex.getLogIdentifier() + " back to running due to rescheduling " - + ((VertexEventTaskReschedule)event).getTaskID()); + + ((VertexEventTaskReschedule) event).getTaskID()); (new TaskRescheduledTransition()).transition(vertex, event); // inform the DAG that we are re-running vertex.eventHandler.handle(new DAGEventVertexReRunning(vertex.getVertexId())); @@ -3868,7 +3862,7 @@ public VertexState transition(VertexImpl vertex, VertexEvent event) { // terminate any running tasks String diagnosticMsg = vertex.getLogIdentifier() + " failed due to post-commit rescheduling of " - + ((VertexEventTaskReschedule)event).getTaskID(); + + ((VertexEventTaskReschedule) event).getTaskID(); LOG.info(diagnosticMsg); vertex.tryEnactKill(VertexTerminationCause.OWN_TASK_FAILURE, TaskTerminationCause.OWN_TASK_FAILURE); @@ -3886,7 +3880,8 @@ private void commitCompleted(VertexEventCommitCompleted commitCompletedEvent) { } else { String diag = "Commit failed for output:" + commitCompletedEvent.getOutputName() + ", vertexId=" + logIdentifier + ", " - + ExceptionUtils.getStackTrace(commitCompletedEvent.getException());; + + ExceptionUtils.getStackTrace(commitCompletedEvent.getException()); + ; LOG.info(diag); addDiagnostic(diag); trySetTerminationCause(VertexTerminationCause.COMMIT_FAILURE); @@ -3895,11 +3890,11 @@ private void commitCompleted(VertexEventCommitCompleted commitCompletedEvent) { } private static class CommitCompletedTransition implements - MultipleArcTransition { + MultipleArcTransition { @Override public VertexState transition(VertexImpl vertex, VertexEvent event) { - vertex.commitCompleted((VertexEventCommitCompleted)event); + vertex.commitCompleted((VertexEventCommitCompleted) event); return checkCommitsForCompletion(vertex); } } @@ -3918,7 +3913,7 @@ private void addDiagnostic(String diag) { } private static boolean isEventFromVertex(Vertex vertex, - EventMetaData sourceMeta) { + EventMetaData sourceMeta) { if (!sourceMeta.getTaskVertexName().equals(vertex.getName())) { return false; } @@ -3926,7 +3921,7 @@ private static boolean isEventFromVertex(Vertex vertex, } private static void checkEventSourceMetadata(Vertex vertex, - EventMetaData sourceMeta) { + EventMetaData sourceMeta) { assert isEventFromVertex(vertex, sourceMeta); } @@ -3942,8 +3937,8 @@ private static void checkEventSourceMetadata(Vertex vertex, // } // } - private static class RouteEventTransition implements - MultipleArcTransition { + private static class RouteEventTransition implements + MultipleArcTransition { @Override public VertexState transition(VertexImpl vertex, VertexEvent event) { VertexEventRouteEvent rEvent = (VertexEventRouteEvent) event; @@ -3967,10 +3962,10 @@ public VertexState transition(VertexImpl vertex, VertexEvent event) { return vertex.getState(); } } - + @Override public TaskAttemptEventInfo getTaskAttemptTezEvents(TezTaskAttemptID attemptID, - int fromEventId, int preRoutedFromEventId, int maxEvents) { + int fromEventId, int preRoutedFromEventId, int maxEvents) { Task task = getTask(attemptID.getTaskID()); ArrayList events = task.getTaskAttemptTezEvents( attemptID, preRoutedFromEventId, maxEvents); @@ -4004,11 +3999,10 @@ public TaskAttemptEventInfo getTaskAttemptTezEvents(TezTaskAttemptID attemptID, continue; } TezEvent tezEvent = eventInfo.tezEvent; - switch(tezEvent.getEventType()) { - case INPUT_FAILED_EVENT: - case DATA_MOVEMENT_EVENT: - case COMPOSITE_DATA_MOVEMENT_EVENT: - { + switch (tezEvent.getEventType()) { + case INPUT_FAILED_EVENT: + case DATA_MOVEMENT_EVENT: + case COMPOSITE_DATA_MOVEMENT_EVENT: { int srcTaskIndex = eventInfo.eventTaskIndex; Edge srcEdge = eventInfo.eventEdge; PendingEventRouteMetadata pendingRoute = null; @@ -4021,7 +4015,7 @@ public TaskAttemptEventInfo getTaskAttemptTezEvents(TezTaskAttemptID attemptID, // the first event must match the pending route event // the only reason it may not match is if in between rounds that event got // obsoleted - if(tezEvent != pendingRoute.getTezEvent()) { + if (tezEvent != pendingRoute.getTezEvent()) { Preconditions.checkState(firstEventObsoleted); // pending routes can be ignored for obsoleted events pendingRoute = null; @@ -4036,17 +4030,16 @@ public TaskAttemptEventInfo getTaskAttemptTezEvents(TezTaskAttemptID attemptID, } } break; - case ROOT_INPUT_DATA_INFORMATION_EVENT: - { + case ROOT_INPUT_DATA_INFORMATION_EVENT: { InputDataInformationEvent riEvent = (InputDataInformationEvent) tezEvent.getEvent(); if (riEvent.getTargetIndex() == taskIndex) { events.add(tezEvent); } } break; - default: - throw new TezUncheckedException("Unexpected event type for task: " - + tezEvent.getEventType()); + default: + throw new TezUncheckedException("Unexpected event type for task: " + + tezEvent.getEventType()); } if (earlyExit) { break; @@ -4074,9 +4067,9 @@ public TaskAttemptEventInfo getTaskAttemptTezEvents(TezTaskAttemptID attemptID, onDemandRouteEventsReadLock.unlock(); } if (!events.isEmpty()) { - for (int i=(events.size() - 1); i>=0; --i) { + for (int i = (events.size() - 1); i >= 0; --i) { TezEvent lastEvent = events.get(i); - // record the last event sent by the AM to the task + // record the last event sent by the AM to the task EventType lastEventType = lastEvent.getEventType(); // if the following changes then critical path logic/recording may need revision if (lastEventType == EventType.COMPOSITE_DATA_MOVEMENT_EVENT || @@ -4092,28 +4085,26 @@ public TaskAttemptEventInfo getTaskAttemptTezEvents(TezTaskAttemptID attemptID, } private void handleRoutedTezEvents(List tezEvents, boolean isPendingEvents) throws AMUserCodeException { - for(TezEvent tezEvent : tezEvents) { + for (TezEvent tezEvent : tezEvents) { if (LOG.isDebugEnabled()) { LOG.debug("Vertex: " + getLogIdentifier() + " routing event: " + tezEvent.getEventType()); } EventMetaData sourceMeta = tezEvent.getSourceInfo(); - switch(tezEvent.getEventType()) { - case CUSTOM_PROCESSOR_EVENT: - { + switch (tezEvent.getEventType()) { + case CUSTOM_PROCESSOR_EVENT: { // set version as app attempt id ((CustomProcessorEvent) tezEvent.getEvent()).setVersion( - appContext.getApplicationAttemptId().getAttemptId()); + appContext.getApplicationAttemptId().getAttemptId()); // route event to task EventMetaData destinationMeta = tezEvent.getDestinationInfo(); Task targetTask = getTask(destinationMeta.getTaskAttemptID().getTaskID()); targetTask.registerTezEvent(tezEvent); } break; - case INPUT_FAILED_EVENT: - case DATA_MOVEMENT_EVENT: - case COMPOSITE_DATA_MOVEMENT_EVENT: - { + case INPUT_FAILED_EVENT: + case DATA_MOVEMENT_EVENT: + case COMPOSITE_DATA_MOVEMENT_EVENT: { if (isEventFromVertex(this, sourceMeta)) { // event from this vertex. send to destination vertex TezTaskAttemptID srcTaId = sourceMeta.getTaskAttemptID(); @@ -4160,96 +4151,92 @@ private void handleRoutedTezEvents(List tezEvents, boolean isPendingEv } } break; - case ROOT_INPUT_DATA_INFORMATION_EVENT: - { - checkEventSourceMetadata(this, sourceMeta); - if (tasksNotYetScheduled) { - // this is only needed to support mixed mode routing. Else for - // on demand routing events can be directly added to taskEvents - // when legacy routing is removed then pending task events can be - // removed. - pendingTaskEvents.add(tezEvent); - } else { - InputDataInformationEvent riEvent = (InputDataInformationEvent) tezEvent.getEvent(); - Task targetTask = getTask(riEvent.getTargetIndex()); - targetTask.registerTezEvent(tezEvent); - } - } - break; - case VERTEX_MANAGER_EVENT: - { - // VM events on task success only can be changed as part of TEZ-1532 - VertexManagerEvent vmEvent = (VertexManagerEvent) tezEvent.getEvent(); - Vertex target = getDAG().getVertex(vmEvent.getTargetVertexName()); - Preconditions.checkArgument(target != null, - "Event sent to unkown vertex: " + vmEvent.getTargetVertexName()); - TezTaskAttemptID srcTaId = sourceMeta.getTaskAttemptID(); - if (srcTaId.getVertexID().equals(vertexId)) { - // this is the producer tasks' vertex - vmEvent.setProducerAttemptIdentifier( - getTaskAttemptIdentifier(dag.getName(), getName(), srcTaId)); - } - if (target == this) { - if (!vmIsInitialized.get()) { - // The VM hasn't been setup yet, defer event consumption - pendingVmEvents.add(vmEvent); + case ROOT_INPUT_DATA_INFORMATION_EVENT: { + checkEventSourceMetadata(this, sourceMeta); + if (tasksNotYetScheduled) { + // this is only needed to support mixed mode routing. Else for + // on demand routing events can be directly added to taskEvents + // when legacy routing is removed then pending task events can be + // removed. + pendingTaskEvents.add(tezEvent); } else { - vertexManager.onVertexManagerEventReceived(vmEvent); + InputDataInformationEvent riEvent = (InputDataInformationEvent) tezEvent.getEvent(); + Task targetTask = getTask(riEvent.getTargetIndex()); + targetTask.registerTezEvent(tezEvent); } - } else { - checkEventSourceMetadata(this, sourceMeta); - eventHandler.handle(new VertexEventRouteEvent(target - .getVertexId(), Collections.singletonList(tezEvent))); } - } break; - case ROOT_INPUT_INITIALIZER_EVENT: - { - InputInitializerEvent riEvent = (InputInitializerEvent) tezEvent.getEvent(); - Vertex target = getDAG().getVertex(riEvent.getTargetVertexName()); - Preconditions.checkArgument(target != null, - "Event sent to unknown vertex: " + riEvent.getTargetVertexName()); - riEvent.setSourceVertexName(tezEvent.getSourceInfo().getTaskVertexName()); - if (target == this) { - if (rootInputDescriptors == null || - !rootInputDescriptors.containsKey(riEvent.getTargetInputName())) { - throw new TezUncheckedException( - "InputInitializerEvent targeted at unknown initializer on vertex " + - logIdentifier + ", Event=" + riEvent); + case VERTEX_MANAGER_EVENT: { + // VM events on task success only can be changed as part of TEZ-1532 + VertexManagerEvent vmEvent = (VertexManagerEvent) tezEvent.getEvent(); + Vertex target = getDAG().getVertex(vmEvent.getTargetVertexName()); + Preconditions.checkArgument(target != null, + "Event sent to unkown vertex: " + vmEvent.getTargetVertexName()); + TezTaskAttemptID srcTaId = sourceMeta.getTaskAttemptID(); + if (srcTaId.getVertexID().equals(vertexId)) { + // this is the producer tasks' vertex + vmEvent.setProducerAttemptIdentifier( + getTaskAttemptIdentifier(dag.getName(), getName(), srcTaId)); } - if (getState() == VertexState.NEW) { - pendingInitializerEvents.add(tezEvent); - } else if (getState() == VertexState.INITIALIZING) { - rootInputInitializerManager.handleInitializerEvents(Collections.singletonList(tezEvent)); + if (target == this) { + if (!vmIsInitialized.get()) { + // The VM hasn't been setup yet, defer event consumption + pendingVmEvents.add(vmEvent); + } else { + vertexManager.onVertexManagerEventReceived(vmEvent); + } } else { - // Currently, INITED and subsequent states means Initializer complete / failure - if (LOG.isDebugEnabled()) { - LOG.debug("Dropping event" + tezEvent + " since state is not INITIALIZING in " - + getLogIdentifier() + ", state=" + getState()); + checkEventSourceMetadata(this, sourceMeta); + eventHandler.handle(new VertexEventRouteEvent(target + .getVertexId(), Collections.singletonList(tezEvent))); + } + } + break; + case ROOT_INPUT_INITIALIZER_EVENT: { + InputInitializerEvent riEvent = (InputInitializerEvent) tezEvent.getEvent(); + Vertex target = getDAG().getVertex(riEvent.getTargetVertexName()); + Preconditions.checkArgument(target != null, + "Event sent to unknown vertex: " + riEvent.getTargetVertexName()); + riEvent.setSourceVertexName(tezEvent.getSourceInfo().getTaskVertexName()); + if (target == this) { + if (rootInputDescriptors == null || + !rootInputDescriptors.containsKey(riEvent.getTargetInputName())) { + throw new TezUncheckedException( + "InputInitializerEvent targeted at unknown initializer on vertex " + + logIdentifier + ", Event=" + riEvent); } + if (getState() == VertexState.NEW) { + pendingInitializerEvents.add(tezEvent); + } else if (getState() == VertexState.INITIALIZING) { + rootInputInitializerManager.handleInitializerEvents(Collections.singletonList(tezEvent)); + } else { + // Currently, INITED and subsequent states means Initializer complete / failure + if (LOG.isDebugEnabled()) { + LOG.debug("Dropping event" + tezEvent + " since state is not INITIALIZING in " + + getLogIdentifier() + ", state=" + getState()); + } + } + } else { + checkEventSourceMetadata(this, sourceMeta); + eventHandler.handle(new VertexEventRouteEvent(target.getVertexId(), + Collections.singletonList(tezEvent))); } - } else { - checkEventSourceMetadata(this, sourceMeta); - eventHandler.handle(new VertexEventRouteEvent(target.getVertexId(), - Collections.singletonList(tezEvent))); } - } break; - case INPUT_READ_ERROR_EVENT: - { + case INPUT_READ_ERROR_EVENT: { checkEventSourceMetadata(this, sourceMeta); Edge srcEdge = sourceVertices.get(this.getDAG().getVertex( sourceMeta.getEdgeVertexName())); srcEdge.sendTezEventToSourceTasks(tezEvent); } break; - default: - throw new TezUncheckedException("Unhandled tez event type: " - + tezEvent.getEventType()); + default: + throw new TezUncheckedException("Unhandled tez event type: " + + tezEvent.getEventType()); } } } - + private void processOnDemandEvent(TezEvent tezEvent, Edge srcEdge, int srcTaskIndex) { onDemandRouteEventsWriteLock.lock(); try { @@ -4263,12 +4250,12 @@ private void processOnDemandEvent(TezEvent tezEvent, Edge srcEdge, int srcTaskIn onDemandRouteEvents.add(new EventInfo(tezEvent, srcEdge, srcTaskIndex)); if (tezEvent.getEventType() == EventType.INPUT_FAILED_EVENT) { for (EventInfo eventInfo : onDemandRouteEvents) { - if (eventInfo.eventEdge == srcEdge + if (eventInfo.eventEdge == srcEdge && eventInfo.tezEvent.getSourceInfo().getTaskAttemptID().equals( - tezEvent.getSourceInfo().getTaskAttemptID()) + tezEvent.getSourceInfo().getTaskAttemptID()) && (eventInfo.tezEvent.getEventType() == EventType.DATA_MOVEMENT_EVENT - || eventInfo.tezEvent - .getEventType() == EventType.COMPOSITE_DATA_MOVEMENT_EVENT)) { + || eventInfo.tezEvent + .getEventType() == EventType.COMPOSITE_DATA_MOVEMENT_EVENT)) { // any earlier data movement events from the same source // edge+task // can be obsoleted by an input failed event from the @@ -4352,7 +4339,7 @@ private org.apache.tez.dag.api.event.VertexState convertInternalState(VertexStat } } - private static class VertexCommitCallback implements FutureCallback{ + private static class VertexCommitCallback implements FutureCallback { private String outputName; private VertexImpl vertex; @@ -4373,7 +4360,6 @@ public void onFailure(Throwable t) { vertex.getEventHandler().handle( new VertexEventCommitCompleted(vertex.vertexId, outputName, false, t)); } - } @Override @@ -4398,7 +4384,8 @@ public void setOutputVertices(Map outVertices) { addIO(vertex.getName()); } } finally { - writeLock.unlock();; + writeLock.unlock(); + ; } } @@ -4422,7 +4409,7 @@ public void setAdditionalInputs(List inputs) { this.rootInputSpecs.put(input.getName(), DEFAULT_ROOT_INPUT_SPECS); } } - + // not taking a lock by design. Speculator callbacks to the vertex will take locks if needed @Override public void handleSpeculatorEvent(SpeculatorEvent event) { @@ -4473,7 +4460,7 @@ public void setAdditionalOutputs(List outputs) { @Nullable @Override public Map> - getAdditionalInputs() { + getAdditionalInputs() { readLock.lock(); try { return this.rootInputDescriptors; @@ -4485,7 +4472,7 @@ public void setAdditionalOutputs(List outputs) { @Nullable @Override public Map> - getAdditionalOutputs() { + getAdditionalOutputs() { return this.additionalOutputs; } @@ -4524,7 +4511,7 @@ public Map getInputVertices() { public Map getOutputVertices() { return Collections.unmodifiableMap(this.targetVertices); } - + @Override public VertexStatistics getStatistics() { readLock.lock(); @@ -4571,7 +4558,7 @@ public Resource getTaskResource() { readLock.unlock(); } } - + void addIO(String name) { ioIndices.put(StringInterner.intern(name), ioIndices.size()); } @@ -4601,17 +4588,17 @@ public List getInputSpecList(int taskIndex) throws AMUserCodeExceptio + (rootInputDescriptors == null ? 0 : rootInputDescriptors.size())); if (rootInputDescriptors != null) { for (Entry> - rootInputDescriptorEntry : rootInputDescriptors.entrySet()) { + rootInputDescriptorEntry : rootInputDescriptors.entrySet()) { inputSpecList.add(new InputSpec(rootInputDescriptorEntry.getKey(), rootInputDescriptorEntry.getValue().getIODescriptor(), rootInputSpecs.get( - rootInputDescriptorEntry.getKey()).getNumPhysicalInputsForWorkUnit(taskIndex))); + rootInputDescriptorEntry.getKey()).getNumPhysicalInputsForWorkUnit(taskIndex))); } } } finally { readLock.unlock(); } - for(Vertex vertex : getInputVertices().keySet()) { + for (Vertex vertex : getInputVertices().keySet()) { /** * It is possible that setParallelism is in the middle of processing in target vertex with * its write lock. So we need to get inputspec by acquiring read lock in target vertex to @@ -4655,7 +4642,7 @@ public List getOutputSpecList(int taskIndex) throws AMUserCodeExcept readLock.unlock(); } - for(Vertex vertex : targetVertices.keySet()) { + for (Vertex vertex : targetVertices.keySet()) { /** * It is possible that setParallelism (which could change numTasks) is in the middle of * processing in target vertex with its write lock. So we need to get outputspec by @@ -4692,7 +4679,6 @@ private InputSpec getDestinationSpecFor(VertexImpl vertex, int taskIndex) throws } } - @Override public List getGroupInputSpecList() { readLock.lock(); @@ -4722,9 +4708,8 @@ public boolean isVertexInitSkipped() { return isVertexInitSkipped; } - private static void logLocationHints(String vertexName, - VertexLocationHint locationHint) { + VertexLocationHint locationHint) { if (locationHint == null) { LOG.debug("No Vertex LocationHint specified for vertex=" + vertexName); return; @@ -4770,7 +4755,7 @@ private static void logLocationHints(String vertexName, } /** - * This is for recovery when VertexReconfigureDoneEvent is seen. + * This is for recovery when VertexReconfigureDoneEvent is seen. */ public static class NoOpVertexManager extends VertexManagerPlugin { @@ -4811,7 +4796,7 @@ public void onVertexStarted(List completions) int numTasks = getContext().getVertexNumTasks(getContext().getVertexName()); LOG.debug("Schedule all the tasks, numTask={}", numTasks); List tasks = new ArrayList(); - for (int i=0;i events) throws Exception { + InputDescriptor inputDescriptor, List events) throws Exception { if (LOG.isDebugEnabled()) { LOG.debug("onRootVertexInitialized is invoked in NoOpVertexManager, vertex=" + getContext().getVertexName()); } @@ -4949,42 +4934,47 @@ public boolean getTaskRescheduleRelaxedLocality() { /** * @return maxAllowedOutputFailures. */ - @Override public int getMaxAllowedOutputFailures() { + @Override + public int getMaxAllowedOutputFailures() { return maxAllowedOutputFailures; } /** * @return maxAllowedOutputFailuresFraction. */ - @Override public double getMaxAllowedOutputFailuresFraction() { + @Override + public double getMaxAllowedOutputFailuresFraction() { return maxAllowedOutputFailuresFraction; } /** * @return maxAllowedTimeForTaskReadErrorSec. */ - @Override public int getMaxAllowedTimeForTaskReadErrorSec() { + @Override + public int getMaxAllowedTimeForTaskReadErrorSec() { return maxAllowedTimeForTaskReadErrorSec; } /** * @return maxAllowedDownstreamHostsReportingFetchFailure. */ - @Override public double getMaxAllowedDownstreamHostFailuresFraction() { + @Override + public double getMaxAllowedDownstreamHostFailuresFraction() { return maxAllowedDownstreamHostFailuresFraction; } } @Override - public AbstractService getSpeculator() { return speculator; } + public AbstractService getSpeculator() {return speculator;} @Override - public Map> getDownstreamBlamingHosts(){ + public Map> getDownstreamBlamingHosts() { return downstreamBlamingHosts; } /** * Initialize context from vertex shuffle deletion. + * * @param deletionHeight */ public void initShuffleDeletionContext(int deletionHeight) { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java index c3d49ea840..6d4424bf5d 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexManager.java @@ -1,25 +1,23 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.impl; - - import java.lang.reflect.UndeclaredThrowableException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; @@ -99,7 +97,7 @@ public class VertexManager { final AppContext appContext; final BlockingQueue rootInputInitEventQueue; final StateChangeNotifier stateChangeNotifier; - + private final ListeningExecutorService execService; private final LinkedBlockingQueue eventQueue; private final AtomicBoolean eventInFlight; @@ -124,13 +122,13 @@ private void checkAndThrowIfDone() { throw new TezUncheckedException("Cannot invoke context methods after throwing an exception"); } } - + @Override public synchronized Map getInputVertexEdgeProperties() { checkAndThrowIfDone(); Map inputs = managedVertex.getInputVertices(); Map vertexEdgeMap = - Maps.newHashMapWithExpectedSize(inputs.size()); + Maps.newHashMapWithExpectedSize(inputs.size()); for (Map.Entry entry : inputs.entrySet()) { vertexEdgeMap.put(entry.getKey().getName(), entry.getValue().getEdgeProperty()); } @@ -142,17 +140,17 @@ public synchronized Map getOutputVertexEdgeProperties() { checkAndThrowIfDone(); Map outputs = managedVertex.getOutputVertices(); Map vertexEdgeMap = - Maps.newHashMapWithExpectedSize(outputs.size()); + Maps.newHashMapWithExpectedSize(outputs.size()); for (Map.Entry entry : outputs.entrySet()) { vertexEdgeMap.put(entry.getKey().getName(), entry.getValue().getEdgeProperty()); } return vertexEdgeMap; } - + @Override public synchronized VertexStatistics getVertexStatistics(String vertexName) { checkAndThrowIfDone(); - return appContext.getCurrentDAG().getVertex(vertexName).getStatistics(); + return appContext.getCurrentDAG().getVertex(vertexName).getStatistics(); } @Override @@ -169,8 +167,8 @@ public synchronized int getVertexNumTasks(String vertexName) { @Override public synchronized void setVertexParallelism(int parallelism, VertexLocationHint vertexLocationHint, - Map sourceEdgeManagers, - Map rootInputSpecUpdate) { + Map sourceEdgeManagers, + Map rootInputSpecUpdate) { checkAndThrowIfDone(); try { managedVertex.setParallelism(parallelism, vertexLocationHint, sourceEdgeManagers, @@ -179,11 +177,11 @@ public synchronized void setVertexParallelism(int parallelism, VertexLocationHin throw new TezUncheckedException(e); } } - + @Override public synchronized void reconfigureVertex(int parallelism, VertexLocationHint vertexLocationHint, - Map sourceEdgeProperties, - Map rootInputSpecUpdate) { + Map sourceEdgeProperties, + Map rootInputSpecUpdate) { checkAndThrowIfDone(); try { managedVertex.reconfigureVertex(parallelism, vertexLocationHint, sourceEdgeProperties, @@ -192,11 +190,11 @@ public synchronized void reconfigureVertex(int parallelism, VertexLocationHint v throw new TezUncheckedException(e); } } - + @Override public synchronized void reconfigureVertex(int parallelism, - @Nullable VertexLocationHint locationHint, - @Nullable Map sourceEdgeProperties) { + @Nullable VertexLocationHint locationHint, + @Nullable Map sourceEdgeProperties) { checkAndThrowIfDone(); try { managedVertex.reconfigureVertex(parallelism, locationHint, sourceEdgeProperties); @@ -204,11 +202,11 @@ public synchronized void reconfigureVertex(int parallelism, throw new TezUncheckedException(e); } } - + @Override public void reconfigureVertex(@Nullable Map rootInputSpecUpdate, - @Nullable VertexLocationHint locationHint, - int parallelism) { + @Nullable VertexLocationHint locationHint, + int parallelism) { checkAndThrowIfDone(); try { managedVertex.reconfigureVertex(rootInputSpecUpdate, parallelism, locationHint); @@ -222,7 +220,7 @@ public synchronized void scheduleTasks(List tasks) { checkAndThrowIfDone(); managedVertex.scheduleTasks(tasks); } - + @Override public synchronized void scheduleVertexTasks(List tasks) { checkAndThrowIfDone(); @@ -255,7 +253,7 @@ public synchronized UserPayload getUserPayload() { @Override public synchronized void addRootInputEvents(final String inputName, - Collection events) { + Collection events) { checkAndThrowIfDone(); verifyIsRootInput(inputName); final long currTime = appContext.getClock().getTime(); @@ -271,7 +269,7 @@ public TezEvent apply(InputDataInformationEvent riEvent) { if (LOG.isDebugEnabled()) { LOG.debug("vertex:" + managedVertex.getLogIdentifier() + "; Added " + events.size() + " for input " + - "name " + inputName); + "name " + inputName); } rootInputInitEventQueue.addAll(tezEvents); // Recovery handling is taken care of by the Vertex. @@ -281,8 +279,8 @@ public TezEvent apply(InputDataInformationEvent riEvent) { public void sendEventToProcessor(Collection events, int taskId) { checkAndThrowIfDone(); Preconditions.checkArgument(taskId >= 0 && taskId < managedVertex.getTotalTasks(), - "Invalid taskId " + taskId + "; " + "There are " + managedVertex.getTotalTasks() - + " tasks in total."); + "Invalid taskId " + taskId + "; " + "There are " + managedVertex.getTotalTasks() + + " tasks in total."); if (events != null && events.size() > 0) { List tezEvents = new ArrayList<>(); @@ -291,13 +289,13 @@ public void sendEventToProcessor(Collection events, int ta // use dummy task attempt id since this is not an task attempt specific event and task // attempt id won't be used anyway EventMetaData destinationMeta = new EventMetaData(EventProducerConsumerType.PROCESSOR, - managedVertex.getName(), managedVertex.getName(), - TezTaskAttemptID.getInstance(managedVertex.getTask(taskId).getTaskID(), -1)); + managedVertex.getName(), managedVertex.getName(), + TezTaskAttemptID.getInstance(managedVertex.getTask(taskId).getTaskID(), -1)); tezEvent.setDestinationInfo(destinationMeta); tezEvents.add(tezEvent); } appContext.getEventHandler().handle( - new VertexEventRouteEvent(managedVertex.getVertexId(), tezEvents)); + new VertexEventRouteEvent(managedVertex.getVertexId(), tezEvents)); } } @@ -350,7 +348,7 @@ public synchronized int getNumClusterNodes() { @Override public synchronized void registerForVertexStateUpdates(String vertexName, Set stateSet) { checkAndThrowIfDone(); - synchronized(notificationRegisteredVertices) { + synchronized (notificationRegisteredVertices) { notificationRegisteredVertices.add(vertexName); } stateChangeNotifier.registerForVertexUpdates(vertexName, stateSet, this); @@ -361,10 +359,9 @@ private void unregisterForVertexStateUpdates() { for (String vertexName : notificationRegisteredVertices) { stateChangeNotifier.unregisterForVertexUpdates(vertexName, this); } - } } - + boolean isComplete() { return (isComplete.get() == true); } @@ -407,11 +404,10 @@ public void onStateUpdated(VertexStateUpdate event) { // no need to synchronize this. similar to other external notification methods enqueueAndScheduleNextEvent(new VertexManagerEventOnVertexStateUpdate(event)); } - } public VertexManager(VertexManagerPluginDescriptor pluginDesc, UserGroupInformation dagUgi, - Vertex managedVertex, AppContext appContext, StateChangeNotifier stateChangeNotifier) throws TezException { + Vertex managedVertex, AppContext appContext, StateChangeNotifier stateChangeNotifier) throws TezException { Objects.requireNonNull(pluginDesc, "pluginDesc is null"); Objects.requireNonNull(managedVertex, "managedVertex is null"); Objects.requireNonNull(appContext, "appContext is null"); @@ -423,12 +419,12 @@ public VertexManager(VertexManagerPluginDescriptor pluginDesc, UserGroupInformat this.stateChangeNotifier = stateChangeNotifier; // don't specify the size of rootInputInitEventQueue, otherwise it will fail when addAll this.rootInputInitEventQueue = new LinkedBlockingQueue(); - + pluginContext = new VertexManagerPluginContextImpl(); payload = pluginDesc.getUserPayload(); pluginFailed = new AtomicBoolean(false); plugin = ReflectionUtils.createClazzInstance(pluginDesc.getClassName(), - new Class[] { VertexManagerPluginContext.class }, new Object[] { pluginContext }); + new Class[]{VertexManagerPluginContext.class}, new Object[]{pluginContext}); execService = appContext.getExecService(); eventQueue = new LinkedBlockingQueue(); eventInFlight = new AtomicBoolean(false); @@ -450,7 +446,7 @@ public void initialize() throws AMUserCodeException { throw new AMUserCodeException(Source.VertexManager, e); } } - + private boolean pluginInvocationAllowed(String msg) { if (pluginFailed.get()) { if (LOG.isDebugEnabled()) { @@ -460,13 +456,13 @@ private boolean pluginInvocationAllowed(String msg) { } if (pluginContext.isComplete()) { if (LOG.isDebugEnabled()) { - LOG.debug(msg+ " . Manager complete. Not scheduling event. Vertex=" + managedVertex.getLogIdentifier()); + LOG.debug(msg + " . Manager complete. Not scheduling event. Vertex=" + managedVertex.getLogIdentifier()); } return false; } return true; } - + private void enqueueAndScheduleNextEvent(VertexManagerEvent e) { if (!pluginInvocationAllowed("Dropping event")) { return; @@ -474,7 +470,7 @@ private void enqueueAndScheduleNextEvent(VertexManagerEvent e) { eventQueue.add(e); tryScheduleNextEvent(); } - + private void tryScheduleNextEvent() { if (!pluginInvocationAllowed("Not scheduling")) { return; @@ -517,7 +513,7 @@ public void onVertexManagerEventReceived( } public void onRootVertexInitialized(String inputName, - InputDescriptor inputDescriptor, List events) throws AMUserCodeException { + InputDescriptor inputDescriptor, List events) throws AMUserCodeException { if (LOG.isDebugEnabled()) { LOG.debug("vertex:" + managedVertex.getLogIdentifier() + "; enqueueing onRootVertexInitialized" + " on input:" + inputName + ", current task events size is " + rootInputInitEventQueue.size()); @@ -551,7 +547,7 @@ public void onFailure(Throwable t) { sendInternalError(e); } } - + @Override public void onSuccess(Void result) { try { @@ -562,10 +558,10 @@ public void onSuccess(Void result) { sendInternalError(e); } } - + protected void onSuccessDerived(Void result) { } - + private void sendInternalError(Exception e) { // fail the DAG so that we dont hang // state change must be triggered via an event transition @@ -573,10 +569,10 @@ private void sendInternalError(Exception e) { appContext.getEventHandler().handle( (new DAGEventInternalError(managedVertex.getVertexId().getDAGID(), "Error in VertexManager for vertex: " + managedVertex.getLogIdentifier() - + ", error=" + ExceptionUtils.getStackTrace(e)))); + + ", error=" + ExceptionUtils.getStackTrace(e)))); } } - + private class VertexManagerRootInputInitializedCallback extends VertexManagerCallback { @Override @@ -592,10 +588,10 @@ protected void onSuccessDerived(Void result) { new VertexEventInputDataInformation(managedVertex.getVertexId(), resultEvents)); } } - + class VertexManagerEventOnVertexStateUpdate extends VertexManagerEvent { private final VertexStateUpdate event; - + public VertexManagerEventOnVertexStateUpdate(VertexStateUpdate event) { this.event = event; } @@ -604,58 +600,54 @@ public VertexManagerEventOnVertexStateUpdate(VertexStateUpdate event) { public void invoke() throws Exception { plugin.onVertexStateUpdated(event); } - } - + class VertexManagerEventOnVertexStarted extends VertexManagerEvent { private final List pluginCompletions; public VertexManagerEventOnVertexStarted(List pluginCompletions) { this.pluginCompletions = pluginCompletions; } - + @Override public void invoke() throws Exception { plugin.onVertexStarted(pluginCompletions); } - } - + class VertexManagerEventSourceTaskCompleted extends VertexManagerEvent { private final TaskAttemptIdentifier attempt; - + public VertexManagerEventSourceTaskCompleted(TaskAttemptIdentifier attempt) { this.attempt = attempt; } - + @Override public void invoke() throws Exception { - plugin.onSourceTaskCompleted(attempt); + plugin.onSourceTaskCompleted(attempt); } - } - + class VertexManagerEventReceived extends VertexManagerEvent { private final org.apache.tez.runtime.api.events.VertexManagerEvent vmEvent; - + public VertexManagerEventReceived(org.apache.tez.runtime.api.events.VertexManagerEvent vmEvent) { this.vmEvent = vmEvent; } - + @Override public void invoke() throws Exception { plugin.onVertexManagerEventReceived(vmEvent); } - } - + class VertexManagerEventRootInputInitialized extends VertexManagerEvent { private final String inputName; private final InputDescriptor inputDescriptor; private final List events; - + public VertexManagerEventRootInputInitialized(String inputName, - InputDescriptor inputDescriptor, List events) { + InputDescriptor inputDescriptor, List events) { super(new VertexManagerRootInputInitializedCallback()); this.inputName = inputName; this.inputDescriptor = inputDescriptor; @@ -666,13 +658,13 @@ public VertexManagerEventRootInputInitialized(String inputName, public void invoke() throws Exception { plugin.onRootVertexInitialized(inputName, inputDescriptor, events); } - } - + abstract class VertexManagerEvent extends CallableEvent { public VertexManagerEvent() { this(VM_CALLBACK); } + public VertexManagerEvent(VertexManagerCallback callback) { super(callback); } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexShuffleDataDeletionContext.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexShuffleDataDeletionContext.java index 4ffdf11231..4871e85bff 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexShuffleDataDeletionContext.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexShuffleDataDeletionContext.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexStats.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexStats.java index c7bccd1783..c75a5e72d9 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexStats.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexStats.java @@ -82,7 +82,7 @@ public Set getLongestDurationTasks() { void updateStats(TaskReport taskReport) { if (firstTaskStartTime == -1 - || firstTaskStartTime >= taskReport.getStartTime()) { + || firstTaskStartTime >= taskReport.getStartTime()) { if (firstTaskStartTime != taskReport.getStartTime()) { firstTasksToStart.clear(); } @@ -91,7 +91,7 @@ void updateStats(TaskReport taskReport) { } if ((taskReport.getFinishTime() > 0) && (lastTaskFinishTime == -1 - || lastTaskFinishTime <= taskReport.getFinishTime())) { + || lastTaskFinishTime <= taskReport.getFinishTime())) { if (lastTaskFinishTime != taskReport.getFinishTime()) { lastTasksToFinish.clear(); } @@ -113,7 +113,7 @@ void updateStats(TaskReport taskReport) { ++numSuccessfulTasks; if (minTaskDuration == -1 - || minTaskDuration >= taskDuration) { + || minTaskDuration >= taskDuration) { if (minTaskDuration != taskDuration) { shortestDurationTasks.clear(); } @@ -121,7 +121,7 @@ void updateStats(TaskReport taskReport) { shortestDurationTasks.add(taskReport.getTaskId()); } if (maxTaskDuration == -1 - || maxTaskDuration <= taskDuration) { + || maxTaskDuration <= taskDuration) { if (maxTaskDuration != taskDuration) { longestDurationTasks.clear(); } @@ -129,12 +129,12 @@ void updateStats(TaskReport taskReport) { longestDurationTasks.add(taskReport.getTaskId()); } - avgTaskDuration = ((avgTaskDuration * (numSuccessfulTasks-1)) + taskDuration) - /numSuccessfulTasks; + avgTaskDuration = ((avgTaskDuration * (numSuccessfulTasks - 1)) + taskDuration) + / numSuccessfulTasks; } private void appendTaskIdSet(StringBuilder sb, - Set taskIDs) { + Set taskIDs) { sb.append("[ "); boolean first = true; if (taskIDs != null) { @@ -154,20 +154,19 @@ private void appendTaskIdSet(StringBuilder sb, public String toString() { StringBuilder sb = new StringBuilder(); sb.append("firstTaskStartTime=").append(firstTaskStartTime) - .append(", firstTasksToStart="); + .append(", firstTasksToStart="); appendTaskIdSet(sb, firstTasksToStart); sb.append(", lastTaskFinishTime=").append(lastTaskFinishTime) - .append(", lastTasksToFinish="); + .append(", lastTasksToFinish="); appendTaskIdSet(sb, lastTasksToFinish); sb.append(", minTaskDuration=").append(minTaskDuration) - .append(", maxTaskDuration=").append(maxTaskDuration) - .append(", avgTaskDuration=").append(avgTaskDuration) - .append(", numSuccessfulTasks=").append(numSuccessfulTasks) - .append(", shortestDurationTasks="); + .append(", maxTaskDuration=").append(maxTaskDuration) + .append(", avgTaskDuration=").append(avgTaskDuration) + .append(", numSuccessfulTasks=").append(numSuccessfulTasks) + .append(", shortestDurationTasks="); appendTaskIdSet(sb, shortestDurationTasks); sb.append(", longestDurationTasks="); appendTaskIdSet(sb, longestDurationTasks); return sb.toString(); } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/package-info.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/package-info.java index 0f8c21df01..2b56e3bb04 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/package-info.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/package-info.java @@ -17,4 +17,5 @@ */ @InterfaceAudience.Private package org.apache.tez.dag.app.dag.impl; + import org.apache.hadoop.classification.InterfaceAudience; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/package-info.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/package-info.java index abf211530c..85a9bbf6aa 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/package-info.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/package-info.java @@ -17,4 +17,5 @@ */ @InterfaceAudience.Private package org.apache.tez.dag.app.dag; + import org.apache.hadoop.classification.InterfaceAudience; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/DataStatistics.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/DataStatistics.java index bbfb950741..205db14c36 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/DataStatistics.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/DataStatistics.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.speculation.legacy; @@ -54,7 +54,7 @@ synchronized void updateStatistics(double old, double update) { public synchronized double mean() { // when no data then mean estimate should be large //return count == 0 ? 0.0 : sum/count; - return count == 0 ? Long.MAX_VALUE : sum/count; + return count == 0 ? Long.MAX_VALUE : sum / count; } public synchronized double var() { @@ -63,7 +63,7 @@ public synchronized double var() { return 0.0; } double mean = mean(); - return Math.max((sumSquares/count) - mean * mean, 0.0d); + return Math.max((sumSquares / count) - mean * mean, 0.0d); } public synchronized double std() { @@ -104,4 +104,4 @@ public String toString() { + ", sumSquares is " + sumSquares + " mean is " + mean() + " std() is " + std() + ", meanCI() is " + meanCI(); } -} \ No newline at end of file +} diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/LegacySpeculator.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/LegacySpeculator.java index 940bb23025..9ddab0fdae 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/LegacySpeculator.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/LegacySpeculator.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.speculation.legacy; @@ -27,8 +27,10 @@ import java.util.concurrent.ConcurrentMap; import com.google.common.annotations.VisibleForTesting; + import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; + import org.apache.hadoop.service.AbstractService; import org.apache.tez.common.ProgressHelper; import org.apache.tez.dag.api.TezConfiguration; @@ -59,7 +61,7 @@ * successive speculations. */ public class LegacySpeculator extends AbstractService { - + private static final long ON_SCHEDULE = Long.MIN_VALUE; private static final long ALREADY_SPECULATING = Long.MIN_VALUE + 1; private static final long TOO_NEW = Long.MIN_VALUE + 2; @@ -72,7 +74,7 @@ public class LegacySpeculator extends AbstractService { private final double proportionRunningTasksSpeculatable; private final double proportionTotalTasksSpeculatable; - private final int minimumAllowedSpeculativeTasks; + private final int minimumAllowedSpeculativeTasks; private static final int VERTEX_SIZE_THRESHOLD_FOR_TIMEOUT_SPECULATION = 1; private static final Logger LOG = LoggerFactory.getLogger(LegacySpeculator.class); @@ -84,7 +86,7 @@ public class LegacySpeculator extends AbstractService { // that we can aggressively speculate instead of waiting for task-timeout. private final ConcurrentMap runningTaskAttemptStatistics = new ConcurrentHashMap(); + TaskAttemptHistoryStatistics>(); // Regular heartbeat from tasks is every 3 secs. So if we don't get a // heartbeat in 9 secs (3 heartbeats), we simulate a heartbeat with no change // in progress. @@ -100,19 +102,19 @@ public class LegacySpeculator extends AbstractService { private volatile boolean stopped = false; @VisibleForTesting - public int getMinimumAllowedSpeculativeTasks() { return minimumAllowedSpeculativeTasks;} + public int getMinimumAllowedSpeculativeTasks() {return minimumAllowedSpeculativeTasks;} @VisibleForTesting - public double getProportionTotalTasksSpeculatable() { return proportionTotalTasksSpeculatable;} + public double getProportionTotalTasksSpeculatable() {return proportionTotalTasksSpeculatable;} @VisibleForTesting - public double getProportionRunningTasksSpeculatable() { return proportionRunningTasksSpeculatable;} + public double getProportionRunningTasksSpeculatable() {return proportionRunningTasksSpeculatable;} @VisibleForTesting - public long getSoonestRetryAfterNoSpeculate() { return soonestRetryAfterNoSpeculate;} + public long getSoonestRetryAfterNoSpeculate() {return soonestRetryAfterNoSpeculate;} @VisibleForTesting - public long getSoonestRetryAfterSpeculate() { return soonestRetryAfterSpeculate;} + public long getSoonestRetryAfterSpeculate() {return soonestRetryAfterSpeculate;} public LegacySpeculator(Configuration conf, AppContext context, Vertex vertex) { this(conf, context.getClock(), vertex); @@ -121,7 +123,7 @@ public LegacySpeculator(Configuration conf, AppContext context, Vertex vertex) { public LegacySpeculator(Configuration conf, Clock clock, Vertex vertex) { this(conf, getEstimator(conf, vertex), clock, vertex); } - + static private TaskRuntimeEstimator getEstimator (Configuration conf, Vertex vertex) { TaskRuntimeEstimator estimator; @@ -187,29 +189,29 @@ public boolean isStarted() { // However, it's public because we do use it in the test cases. // Normally we figure out our own estimator. public LegacySpeculator - (Configuration conf, TaskRuntimeEstimator estimator, Clock clock, Vertex vertex) { + (Configuration conf, TaskRuntimeEstimator estimator, Clock clock, Vertex vertex) { super(LegacySpeculator.class.getName()); this.vertex = vertex; this.estimator = estimator; this.clock = clock; taskTimeout = conf.getLong( - TezConfiguration.TEZ_AM_LEGACY_SPECULATIVE_SINGLE_TASK_VERTEX_TIMEOUT, - TezConfiguration.TEZ_AM_LEGACY_SPECULATIVE_SINGLE_TASK_VERTEX_TIMEOUT_DEFAULT); + TezConfiguration.TEZ_AM_LEGACY_SPECULATIVE_SINGLE_TASK_VERTEX_TIMEOUT, + TezConfiguration.TEZ_AM_LEGACY_SPECULATIVE_SINGLE_TASK_VERTEX_TIMEOUT_DEFAULT); soonestRetryAfterNoSpeculate = conf.getLong( - TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_NO_SPECULATE, - TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_NO_SPECULATE_DEFAULT); + TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_NO_SPECULATE, + TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_NO_SPECULATE_DEFAULT); soonestRetryAfterSpeculate = conf.getLong( - TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_SPECULATE, - TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_SPECULATE_DEFAULT); + TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_SPECULATE, + TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_SPECULATE_DEFAULT); proportionRunningTasksSpeculatable = conf.getDouble( - TezConfiguration.TEZ_AM_PROPORTION_RUNNING_TASKS_SPECULATABLE, - TezConfiguration.TEZ_AM_PROPORTION_RUNNING_TASKS_SPECULATABLE_DEFAULT); + TezConfiguration.TEZ_AM_PROPORTION_RUNNING_TASKS_SPECULATABLE, + TezConfiguration.TEZ_AM_PROPORTION_RUNNING_TASKS_SPECULATABLE_DEFAULT); proportionTotalTasksSpeculatable = conf.getDouble( - TezConfiguration.TEZ_AM_PROPORTION_TOTAL_TASKS_SPECULATABLE, - TezConfiguration.TEZ_AM_PROPORTION_TOTAL_TASKS_SPECULATABLE_DEFAULT); + TezConfiguration.TEZ_AM_PROPORTION_TOTAL_TASKS_SPECULATABLE, + TezConfiguration.TEZ_AM_PROPORTION_TOTAL_TASKS_SPECULATABLE_DEFAULT); minimumAllowedSpeculativeTasks = conf.getInt( - TezConfiguration.TEZ_AM_MINIMUM_ALLOWED_SPECULATIVE_TASKS, - TezConfiguration.TEZ_AM_MINIMUM_ALLOWED_SPECULATIVE_TASKS_DEFAULT); + TezConfiguration.TEZ_AM_MINIMUM_ALLOWED_SPECULATIVE_TASKS, + TezConfiguration.TEZ_AM_MINIMUM_ALLOWED_SPECULATIVE_TASKS_DEFAULT); } @Override @@ -256,15 +258,15 @@ public void run() { }; } -/* ************************************************************* */ + /* ************************************************************* */ public void notifyAttemptStarted(TezTaskAttemptID taId, long timestamp) { - estimator.enrollAttempt(taId, timestamp); + estimator.enrollAttempt(taId, timestamp); } public void notifyAttemptStatusUpdate(TezTaskAttemptID taId, - TaskAttemptState reportedState, - long timestamp) { + TaskAttemptState reportedState, + long timestamp) { statusUpdate(taId, reportedState, timestamp); } @@ -277,7 +279,7 @@ public void notifyAttemptStatusUpdate(TezTaskAttemptID taId, * because statuses contain progress. */ private void statusUpdate(TezTaskAttemptID attemptID, - TaskAttemptState reportedState, long timestamp) { + TaskAttemptState reportedState, long timestamp) { TezTaskID taskID = attemptID.getTaskID(); Task task = vertex.getTask(taskID); @@ -297,7 +299,7 @@ private void statusUpdate(TezTaskAttemptID attemptID, } } } - + public void handle(SpeculatorEvent event) { SpeculatorEventTaskAttemptStatusUpdate updateEvent = ((SpeculatorEventTaskAttemptStatusUpdate) event); if (updateEvent.hasJustStarted()) { @@ -308,12 +310,11 @@ public void handle(SpeculatorEvent event) { } } -/* ************************************************************* */ + /* ************************************************************* */ // This is the code section that runs periodically and adds speculations for // those jobs that need them. - // This can return a few magic values for tasks that shouldn't speculate: // returns ON_SCHEDULE if thresholdRuntime(taskID) says that we should not // considering speculating this task @@ -386,18 +387,18 @@ private long speculationValue(Task task, long now, boolean shouldUseTimeout) { long estimatedEndTime = estimatedRunTime + taskAttemptStartTime; long estimatedReplacementEndTime - = now + estimator.newAttemptEstimatedRuntime(); + = now + estimator.newAttemptEstimatedRuntime(); float progress = taskAttempt.getProgress(); TaskAttemptHistoryStatistics data = - runningTaskAttemptStatistics.get(runningTaskAttemptID); + runningTaskAttemptStatistics.get(runningTaskAttemptID); if (data == null) { runningTaskAttemptStatistics.put(runningTaskAttemptID, new TaskAttemptHistoryStatistics(estimatedRunTime, progress, now)); } else { if (estimatedRunTime == data.getEstimatedRunTime() - && progress == data.getProgress()) { + && progress == data.getProgress()) { // Previous stats are same as same stats if (data.notHeartbeatedInAWhile(now) || estimator @@ -491,7 +492,7 @@ int computeSpeculations() { } numberAllowedSpeculativeTasks = (int) Math.max(numberAllowedSpeculativeTasks, - proportionRunningTasksSpeculatable * numberRunningTasks); + proportionRunningTasksSpeculatable * numberRunningTasks); // If we found a speculation target, fire it off if (bestTaskID != null @@ -509,7 +510,7 @@ static class TaskAttemptHistoryStatistics { private long lastHeartBeatTime; public TaskAttemptHistoryStatistics(long estimatedRunTime, float progress, - long nonProgressStartTime) { + long nonProgressStartTime) { this.estimatedRunTime = estimatedRunTime; this.progress = progress; resetHeartBeatTime(nonProgressStartTime); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/LegacyTaskRuntimeEstimator.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/LegacyTaskRuntimeEstimator.java index 14d269cc33..25ce8c7ab1 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/LegacyTaskRuntimeEstimator.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/LegacyTaskRuntimeEstimator.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.speculation.legacy; @@ -41,7 +41,6 @@ public class LegacyTaskRuntimeEstimator extends StartEndTimesBase { @Override public void updateAttempt(TezTaskAttemptID attemptID, TaskAttemptState state, long timestamp) { super.updateAttempt(attemptID, state, timestamp); - Task task = vertex.getTask(attemptID.getTaskID()); @@ -54,7 +53,7 @@ public void updateAttempt(TezTaskAttemptID attemptID, TaskAttemptState state, lo if (taskAttempt == null) { return; } - + float progress = taskAttempt.getProgress(); Long boxedStart = startTimes.get(attemptID); @@ -85,7 +84,6 @@ public void updateAttempt(TezTaskAttemptID attemptID, TaskAttemptState state, lo estimateVarianceContainer = attemptRuntimeEstimateVariances.get(taskAttempt); } - long estimate = -1; long varianceEstimate = -1; @@ -105,7 +103,7 @@ public void updateAttempt(TezTaskAttemptID attemptID, TaskAttemptState state, lo } private long storedPerAttemptValue - (Map data, TezTaskAttemptID attemptID) { + (Map data, TezTaskAttemptID attemptID) { Task task = vertex.getTask(attemptID.getTaskID()); if (task == null) { @@ -121,7 +119,6 @@ public void updateAttempt(TezTaskAttemptID attemptID, TaskAttemptState state, lo AtomicLong estimate = data.get(taskAttempt); return estimate == null ? -1L : estimate.get(); - } @Override diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/SimpleExponentialTaskRuntimeEstimator.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/SimpleExponentialTaskRuntimeEstimator.java index b61f153e6b..d659855206 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/SimpleExponentialTaskRuntimeEstimator.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/SimpleExponentialTaskRuntimeEstimator.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,6 +20,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicReference; + import org.apache.hadoop.conf.Configuration; import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.oldrecords.TaskAttemptState; @@ -84,7 +85,7 @@ private SimpleExponentialSmoothing getForecastEntry( } private void incorporateReading(final TezTaskAttemptID attemptID, - final float newRawData, final long newTimeStamp) { + final float newRawData, final long newTimeStamp) { SimpleExponentialSmoothing foreCastEntry = getForecastEntry(attemptID); if (foreCastEntry == null) { Long tStartTime = startTimes.get(attemptID); @@ -151,7 +152,7 @@ public long newAttemptEstimatedRuntime() { @Override public boolean hasStagnatedProgress(final TezTaskAttemptID id, - final long timeStamp) { + final long timeStamp) { SimpleExponentialSmoothing foreCastEntry = getForecastEntry(id); if (foreCastEntry == null) { return false; @@ -173,11 +174,10 @@ public long runtimeEstimateVariance(final TezTaskAttemptID id) { return 0L; } - @Override public void updateAttempt(final TezTaskAttemptID attemptID, - final TaskAttemptState state, - final long timestamp) { + final TaskAttemptState state, + final long timestamp) { super.updateAttempt(attemptID, state, timestamp); Task task = vertex.getTask(attemptID.getTaskID()); if (task == null) { @@ -191,4 +191,3 @@ public void updateAttempt(final TezTaskAttemptID attemptID, incorporateReading(attemptID, progress, timestamp); } } - diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/StartEndTimesBase.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/StartEndTimesBase.java index 3083986d9c..5b7d2017b2 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/StartEndTimesBase.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/StartEndTimesBase.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.speculation.legacy; @@ -98,8 +98,8 @@ public long newAttemptEstimatedRuntime() { @Override public void updateAttempt(final TezTaskAttemptID attemptID, - final TaskAttemptState state, - final long timestamp) { + final TaskAttemptState state, + final long timestamp) { Task task = vertex.getTask(attemptID.getTaskID()); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/TaskRuntimeEstimator.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/TaskRuntimeEstimator.java index 4f747afc4e..0229330030 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/TaskRuntimeEstimator.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/TaskRuntimeEstimator.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.speculation.legacy; @@ -26,7 +26,7 @@ /** * Estimate the runtime for tasks of a given vertex. - * + * */ public interface TaskRuntimeEstimator { void enrollAttempt(TezTaskAttemptID id, long timestamp); @@ -34,7 +34,7 @@ public interface TaskRuntimeEstimator { long attemptEnrolledTime(TezTaskAttemptID attemptID); void updateAttempt(TezTaskAttemptID taId, - TaskAttemptState reportedState, long timestamp); + TaskAttemptState reportedState, long timestamp); void contextualize(Configuration conf, Vertex vertex); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/forecast/SimpleExponentialSmoothing.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/forecast/SimpleExponentialSmoothing.java index e7b7dcd57c..e3583326b6 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/forecast/SimpleExponentialSmoothing.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/forecast/SimpleExponentialSmoothing.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -60,7 +60,7 @@ public static SimpleExponentialSmoothing createForecast( * @param timeStamp the time stamp */ SimpleExponentialSmoothing(final long ktConstant, final int skipCnt, - final long stagnatedWindow, final long timeStamp) { + final long stagnatedWindow, final long timeStamp) { this.kMinimumReads = skipCnt; this.kStagnatedWindow = stagnatedWindow; this.timeConstant = ktConstant; @@ -86,7 +86,7 @@ private class ForecastRecord { * @param currTimeStamp the curr time stamp */ ForecastRecord(final double currForecast, final double currRawData, - final long currTimeStamp) { + final long currTimeStamp) { this(0.0, currForecast, currRawData, currForecast, currTimeStamp, 0.0, 0); } @@ -102,10 +102,10 @@ private class ForecastRecord { * @param index the index */ ForecastRecord(final double alphaVal, final double currSample, - final double currRawData, - final double currForecast, final long currTimeStamp, - final double accError, - final long index) { + final double currRawData, + final double currForecast, final long currTimeStamp, + final double accError, + final long index) { this.timeStamp = currTimeStamp; this.alpha = alphaVal; this.sample = currSample; @@ -116,12 +116,12 @@ private class ForecastRecord { } private ForecastRecord createForecastRecord(final double alphaVal, - final double currSample, - final double currRawData, - final double currForecast, final long currTimeStamp, - final double accError, - final long index, - final ForecastRecord prev) { + final double currSample, + final double currRawData, + final double currForecast, final long currTimeStamp, + final double accError, + final long index, + final ForecastRecord prev) { ForecastRecord forecastRec = new ForecastRecord(alphaVal, currSample, currRawData, currForecast, currTimeStamp, accError, index); @@ -195,7 +195,7 @@ public boolean isDataStagnated(final long timeStamp) { * @return the double */ static double processRawData(final double oldRawData, final long oldTime, - final double newRawData, final long newTime) { + final double newRawData, final long newTime) { double rate = (newRawData - oldRawData) / (newTime - oldTime); return rate; } @@ -207,7 +207,7 @@ static double processRawData(final double oldRawData, final long oldTime, * @param currRawData the curr raw data */ public void incorporateReading(final long timeStamp, - final double currRawData) { + final double currRawData) { ForecastRecord oldRec = forecastRefEntry.get(); if (oldRec == null) { double oldForecast = @@ -333,4 +333,3 @@ public String toString() { return res; } } - diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/forecast/package-info.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/forecast/package-info.java index 3ed8b6accb..25f2c481b6 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/forecast/package-info.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/speculation/legacy/forecast/package-info.java @@ -17,4 +17,5 @@ */ @InterfaceAudience.Private package org.apache.tez.dag.app.dag.speculation.legacy.forecast; + import org.apache.hadoop.classification.InterfaceAudience; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherManager.java index 65360d6d01..93c654e5a6 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherManager.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherManager.java @@ -67,8 +67,6 @@ public class ContainerLauncherManager extends AbstractService private final AppContext appContext; private final boolean isIncompleteCtor; - - // Accepting conf to setup final parameters, if required. public ContainerLauncherManager(AppContext context, TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, @@ -86,7 +84,6 @@ public ContainerLauncherManager(AppContext context, containerLaunchers = new ContainerLauncherWrapper[containerLauncherDescriptors.size()]; containerLauncherServiceWrappers = new ServicePluginLifecycleAbstractService[containerLauncherDescriptors.size()]; - for (int i = 0; i < containerLauncherDescriptors.size(); i++) { UserPayload userPayload = containerLauncherDescriptors.get(i).getUserPayload(); ContainerLauncherContext containerLauncherContext = @@ -169,7 +166,7 @@ ContainerLauncher createUberContainerLauncher(ContainerLauncherContext container @SuppressWarnings("unchecked") ContainerLauncher createCustomContainerLauncher(ContainerLauncherContext containerLauncherContext, NamedEntityDescriptor containerLauncherDescriptor) - throws TezException { + throws TezException { LOG.info("Creating container launcher {}:{} ", containerLauncherDescriptor.getEntityName(), containerLauncherDescriptor.getClassName()); return ReflectionUtils.createClazzInstance(containerLauncherDescriptor.getClassName(), @@ -179,27 +176,27 @@ ContainerLauncher createCustomContainerLauncher(ContainerLauncherContext contain @Override public void serviceInit(Configuration conf) { - for (int i = 0 ; i < containerLaunchers.length ; i++) { + for (int i = 0; i < containerLaunchers.length; i++) { containerLauncherServiceWrappers[i].init(conf); } } @Override public void serviceStart() { - for (int i = 0 ; i < containerLaunchers.length ; i++) { + for (int i = 0; i < containerLaunchers.length; i++) { containerLauncherServiceWrappers[i].start(); } } @Override public void serviceStop() { - for (int i = 0 ; i < containerLaunchers.length ; i++) { + for (int i = 0; i < containerLaunchers.length; i++) { containerLauncherServiceWrappers[i].stop(); } } public void dagComplete(TezDAGID dag, JobTokenSecretManager secretManager) { - for (int i = 0 ; i < containerLaunchers.length ; i++) { + for (int i = 0; i < containerLaunchers.length; i++) { containerLaunchers[i].dagComplete(dag, secretManager); } } @@ -221,7 +218,6 @@ public void dagSubmitted() { // the context has updated information. } - @Override public void handle(ContainerLauncherEvent event) { int launcherId = event.getLauncherId(); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherWrapper.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherWrapper.java index 4703abe100..e8bf5a69d7 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherWrapper.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherWrapper.java @@ -48,7 +48,7 @@ public ContainerLauncher getContainerLauncher() { public void dagComplete(TezDAGID dag, JobTokenSecretManager jobTokenSecretManager) { if (real instanceof DagContainerLauncher) { - ((DagContainerLauncher)real).dagComplete(dag, jobTokenSecretManager); + ((DagContainerLauncher) real).dagComplete(dag, jobTokenSecretManager); } } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/DagDeleteRunnable.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/DagDeleteRunnable.java index cbea36b86d..7ce39f91f7 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/DagDeleteRunnable.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/DagDeleteRunnable.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/DeletionTracker.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/DeletionTracker.java index 56760c86be..2faf492035 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/DeletionTracker.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/DeletionTracker.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/DeletionTrackerImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/DeletionTrackerImpl.java index 73eaa68e72..2ff7a1d0c7 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/DeletionTrackerImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/DeletionTrackerImpl.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,7 +18,6 @@ package org.apache.tez.dag.app.launcher; - import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -96,7 +95,7 @@ public void taskAttemptFailed(TezTaskAttemptID taskAttemptID, JobTokenSecretMana @Override public void addNodeShufflePort(NodeId nodeId, int port) { if (port != TezRuntimeUtils.INVALID_PORT) { - if(nodeIdShufflePortMap.get(nodeId) == null) { + if (nodeIdShufflePortMap.get(nodeId) == null) { nodeIdShufflePortMap.put(nodeId, port); } } @@ -127,7 +126,7 @@ public void vertexComplete(TezVertexID vertex, JobTokenSecretManager jobTokenSec } if (shufflePort != null) { VertexDeleteRunnable vertexDeleteRunnable = new VertexDeleteRunnable(vertex, jobTokenSecretManager, nodeId, - shufflePort, vertexId, TezRuntimeUtils.getHttpConnectionParams(conf)); + shufflePort, vertexId, TezRuntimeUtils.getHttpConnectionParams(conf)); try { dagCleanupService.submit(vertexDeleteRunnable); } catch (RejectedExecutionException rejectedException) { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java index 47cc9f1325..ab040b83c8 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/LocalContainerLauncher.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -77,7 +77,6 @@ import org.apache.tez.runtime.api.impl.ExecutionContextImpl; import org.apache.tez.runtime.task.TezChild; - /** * Runs the container task locally in a thread. * Since all (sub)tasks share the same local directory, they must be executed @@ -106,7 +105,7 @@ public class LocalContainerLauncher extends DagContainerLauncher { new ConcurrentHashMap<>(); private final ConcurrentHashMap - cacheManagers = new ConcurrentHashMap<>(); + cacheManagers = new ConcurrentHashMap<>(); private final ExecutorService callbackExecutor = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("CallbackExecutor").build()); @@ -114,10 +113,8 @@ public class LocalContainerLauncher extends DagContainerLauncher { private BlockingQueue eventQueue = new LinkedBlockingQueue<>(); private Thread eventHandlingThread; - private ListeningExecutorService taskExecutorService; - public LocalContainerLauncher(ContainerLauncherContext containerLauncherContext, AppContext context, TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, @@ -157,20 +154,20 @@ public LocalContainerLauncher(ContainerLauncherContext containerLauncherContext, } numExecutors = conf.getInt(TezConfiguration.TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS, TezConfiguration.TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS_DEFAULT); - Preconditions.checkState(numExecutors >=1, "Must have at least 1 executor"); + Preconditions.checkState(numExecutors >= 1, "Must have at least 1 executor"); ExecutorService rawExecutor = Executors.newFixedThreadPool(numExecutors, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("LocalTaskExecutionThread #%d") .build()); this.taskExecutorService = MoreExecutors.listeningDecorator(rawExecutor); dagDelete = ShuffleUtils.isTezShuffleHandler(conf) && conf.getBoolean(TezConfiguration.TEZ_AM_DAG_CLEANUP_ON_COMPLETION, - TezConfiguration.TEZ_AM_DAG_CLEANUP_ON_COMPLETION_DEFAULT); + TezConfiguration.TEZ_AM_DAG_CLEANUP_ON_COMPLETION_DEFAULT); vertexDelete = ShuffleUtils.isTezShuffleHandler(conf) && - conf.getInt(TezConfiguration.TEZ_AM_VERTEX_CLEANUP_HEIGHT, - TezConfiguration.TEZ_AM_VERTEX_CLEANUP_HEIGHT_DEFAULT) > 0; + conf.getInt(TezConfiguration.TEZ_AM_VERTEX_CLEANUP_HEIGHT, + TezConfiguration.TEZ_AM_VERTEX_CLEANUP_HEIGHT_DEFAULT) > 0; failedTaskAttemptDelete = ShuffleUtils.isTezShuffleHandler(conf) && conf.getBoolean(TezConfiguration.TEZ_AM_TASK_ATTEMPT_CLEANUP_ON_FAILURE, - TezConfiguration.TEZ_AM_TASK_ATTEMPT_CLEANUP_ON_FAILURE_DEFAULT); + TezConfiguration.TEZ_AM_TASK_ATTEMPT_CLEANUP_ON_FAILURE_DEFAULT); if (dagDelete || vertexDelete || failedTaskAttemptDelete) { String deletionTrackerClassName = conf.get(TezConfiguration.TEZ_AM_DELETION_TRACKER_CLASS, @@ -206,8 +203,6 @@ public void shutdown() throws Exception { } } - - // Thread to monitor the queue of incoming NMCommunicator events private class TezSubTaskRunner implements Runnable { @Override @@ -280,7 +275,7 @@ private void launch(ContainerLaunchRequest event) { tezChild = createTezChild(conf, event.getContainerId(), tokenIdentifier, context.getApplicationAttemptId().getAttemptId(), context.getLocalDirs(), - ((TezTaskCommunicatorImpl)tal.getTaskCommunicator(taskCommId).getTaskCommunicator()).getUmbilical(), + ((TezTaskCommunicatorImpl) tal.getTaskCommunicator(taskCommId).getTaskCommunicator()).getUmbilical(), TezCommonUtils.parseCredentialsBytes(event.getContainerLaunchContext().getTokens().array())); } catch (InterruptedException e) { handleLaunchFailed(e, event.getContainerId()); @@ -392,7 +387,6 @@ private void cleanupCacheFiles(ContainerId container) { } } - //create a SubTask private synchronized Callable createSubTask( final TezChild tezChild, final ContainerId containerId) { @@ -434,7 +428,6 @@ private TezChild createTezChild(Configuration defaultConf, ContainerId container return tezChild; } - @Override public void launchContainer(ContainerLaunchRequest launchRequest) { try { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/TaskAttemptFailedDeleteRunnable.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/TaskAttemptFailedDeleteRunnable.java index 3a9f5345a0..7fb07f8e72 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/TaskAttemptFailedDeleteRunnable.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/TaskAttemptFailedDeleteRunnable.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -39,8 +39,8 @@ class TaskAttemptFailedRunnable implements Runnable { private final HttpConnectionParams httpConnectionParams; TaskAttemptFailedRunnable(NodeId nodeId, int shufflePort, TezTaskAttemptID taskAttemptID, - HttpConnectionParams httpConnectionParams, - JobTokenSecretManager jobTokenSecretMgr) { + HttpConnectionParams httpConnectionParams, + JobTokenSecretManager jobTokenSecretMgr) { this.nodeId = nodeId; this.shufflePort = shufflePort; this.taskAttemptID = taskAttemptID; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/TezContainerLauncherImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/TezContainerLauncherImpl.java index 654224adb5..2bad7c4cc2 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/TezContainerLauncherImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/TezContainerLauncherImpl.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.launcher; @@ -68,7 +68,6 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; - // TODO See what part of this lifecycle and state management can be simplified. // Ideally, no state - only sendStart / sendStop. @@ -83,7 +82,7 @@ public class TezContainerLauncherImpl extends DagContainerLauncher { static final Logger LOG = LoggerFactory.getLogger(TezContainerLauncherImpl.class); private final ConcurrentHashMap containers = - new ConcurrentHashMap<>(); + new ConcurrentHashMap<>(); protected ThreadPoolExecutor launcherPool; protected static final int INITIAL_POOL_SIZE = 10; private final int limitOnPoolSize; @@ -100,11 +99,11 @@ public class TezContainerLauncherImpl extends DagContainerLauncher { private Container getContainer(ContainerOp event) { ContainerId id = event.getBaseOperation().getContainerId(); Container c = containers.get(id); - if(c == null) { + if (c == null) { c = new Container(id, event.getBaseOperation().getNodeId().toString(), event.getBaseOperation().getContainerToken()); Container old = containers.putIfAbsent(id, c); - if(old != null) { + if (old != null) { c = old; } } @@ -113,12 +112,11 @@ private Container getContainer(ContainerOp event) { private void removeContainerIfDone(ContainerId id) { Container c = containers.get(id); - if(c != null && c.isCompletelyDone()) { + if (c != null && c.isCompletelyDone()) { containers.remove(id); } } - private static enum ContainerState { PREP, FAILED, RUNNING, DONE, KILLED_BEFORE_LAUNCH } @@ -131,7 +129,7 @@ private class Container { private Token containerToken; public Container(ContainerId containerID, - String containerMgrAddress, Token containerToken) { + String containerMgrAddress, Token containerToken) { this.state = ContainerState.PREP; this.containerMgrAddress = containerMgrAddress; this.containerID = containerID; @@ -145,7 +143,7 @@ public synchronized boolean isCompletelyDone() { @SuppressWarnings("unchecked") public synchronized void launch(ContainerLaunchRequest event) { LOG.info("Launching " + event.getContainerId()); - if(this.state == ContainerState.KILLED_BEFORE_LAUNCH) { + if (this.state == ContainerState.KILLED_BEFORE_LAUNCH) { state = ContainerState.DONE; sendContainerLaunchFailedMsg(event.getContainerId(), "Container was killed before it was launched"); @@ -160,11 +158,11 @@ public synchronized void launch(ContainerLaunchRequest event) { // Construct the actual Container ContainerLaunchContext containerLaunchContext = - event.getContainerLaunchContext(); + event.getContainerLaunchContext(); // Now launch the actual container StartContainerRequest startRequest = Records - .newRecord(StartContainerRequest.class); + .newRecord(StartContainerRequest.class); startRequest.setContainerToken(event.getContainerToken()); startRequest.setContainerLaunchContext(containerLaunchContext); @@ -216,10 +214,10 @@ public synchronized void launch(ContainerLaunchRequest event) { @SuppressWarnings("unchecked") public synchronized void kill() { - if(isCompletelyDone()) { + if (isCompletelyDone()) { return; } - if(this.state == ContainerState.PREP) { + if (this.state == ContainerState.PREP) { this.state = ContainerState.KILLED_BEFORE_LAUNCH; } else { LOG.info("Stopping " + containerID); @@ -229,22 +227,22 @@ public synchronized void kill() { proxy = getCMProxy(this.containerID, this.containerMgrAddress, this.containerToken); - // kill the remote container if already launched - StopContainersRequest stopRequest = Records + // kill the remote container if already launched + StopContainersRequest stopRequest = Records .newRecord(StopContainersRequest.class); - stopRequest.setContainerIds(Collections.singletonList(containerID)); + stopRequest.setContainerIds(Collections.singletonList(containerID)); - proxy.getContainerManagementProtocol().stopContainers(stopRequest); + proxy.getContainerManagementProtocol().stopContainers(stopRequest); - // If stopContainer returns without an error, assuming the stop made - // it over to the NodeManager. + // If stopContainer returns without an error, assuming the stop made + // it over to the NodeManager. getContext().containerStopRequested(containerID); } catch (Throwable t) { // ignore the cleanup failure String message = "cleanup failed for container " - + this.containerID + " : " - + ExceptionUtils.getStackTrace(t); + + this.containerID + " : " + + ExceptionUtils.getStackTrace(t); getContext().containerStopFailed(containerID, message); LOG.warn(message); this.state = ContainerState.DONE; @@ -298,7 +296,7 @@ public void run() { try { event = eventQueue.take(); } catch (InterruptedException e) { - if(!serviceStopped.get()) { + if (!serviceStopped.get()) { LOG.error("Returning, interrupted : " + e); } return; @@ -342,8 +340,8 @@ public void run() { conf.getBoolean(TezConfiguration.TEZ_AM_DAG_CLEANUP_ON_COMPLETION, TezConfiguration.TEZ_AM_DAG_CLEANUP_ON_COMPLETION_DEFAULT); vertexDelete = ShuffleUtils.isTezShuffleHandler(conf) && - conf.getInt(TezConfiguration.TEZ_AM_VERTEX_CLEANUP_HEIGHT, - TezConfiguration.TEZ_AM_VERTEX_CLEANUP_HEIGHT_DEFAULT) > 0; + conf.getInt(TezConfiguration.TEZ_AM_VERTEX_CLEANUP_HEIGHT, + TezConfiguration.TEZ_AM_VERTEX_CLEANUP_HEIGHT_DEFAULT) > 0; failedTaskAttemptDelete = ShuffleUtils.isTezShuffleHandler(conf) && conf.getBoolean(TezConfiguration.TEZ_AM_TASK_ATTEMPT_CLEANUP_ON_FAILURE, TezConfiguration.TEZ_AM_TASK_ATTEMPT_CLEANUP_ON_FAILURE_DEFAULT); @@ -358,7 +356,7 @@ public void run() { @Override public void shutdown() { - if(!serviceStopped.compareAndSet(false, true)) { + if (!serviceStopped.compareAndSet(false, true)) { LOG.info("Ignoring multiple stops"); return; } @@ -401,7 +399,7 @@ public void run() { LOG.debug("Processing ContainerOperation {}", event); Container c = getContainer(event); - switch(event.getOpType()) { + switch (event.getOpType()) { case LAUNCH_REQUEST: ContainerLaunchRequest launchRequest = event.getLaunchRequest(); c.launch(launchRequest); @@ -428,12 +426,11 @@ public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { @SuppressWarnings("unchecked") void sendContainerLaunchFailedMsg(ContainerId containerId, - String message) { + String message) { LOG.error(message); getContext().containerLaunchFailed(containerId, message); } - @Override public void launchContainer(ContainerLaunchRequest launchRequest) { try { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/VertexDeleteRunnable.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/VertexDeleteRunnable.java index 3bfec9663a..a47d2e211c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/VertexDeleteRunnable.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/VertexDeleteRunnable.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -40,8 +40,8 @@ public class VertexDeleteRunnable implements Runnable { final private HttpConnectionParams httpConnectionParams; VertexDeleteRunnable(TezVertexID vertex, JobTokenSecretManager jobTokenSecretManager, - NodeId nodeId, int shufflePort, String vertexId, - HttpConnectionParams httpConnectionParams) { + NodeId nodeId, int shufflePort, String vertexId, + HttpConnectionParams httpConnectionParams) { this.vertex = vertex; this.jobTokenSecretManager = jobTokenSecretManager; this.nodeId = nodeId; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/package-info.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/package-info.java index 63ad01f940..c7c62f3434 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/package-info.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/package-info.java @@ -17,4 +17,5 @@ */ @InterfaceAudience.Private package org.apache.tez.dag.app.launcher; + import org.apache.hadoop.classification.InterfaceAudience; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/package-info.java b/tez-dag/src/main/java/org/apache/tez/dag/app/package-info.java index a88dfabf38..2dd491880e 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/package-info.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/package-info.java @@ -17,4 +17,5 @@ */ @InterfaceAudience.Private package org.apache.tez.dag.app; + import org.apache.hadoop.classification.InterfaceAudience; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEvent.java index dd9d951218..20761a0fc2 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEvent.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -19,7 +19,6 @@ import org.apache.hadoop.yarn.event.AbstractEvent; - public class AMSchedulerEvent extends AbstractEvent { private final int schedulerId; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventDeallocateContainer.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventDeallocateContainer.java index d1ca99e4a9..820e768441 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventDeallocateContainer.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventDeallocateContainer.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -28,9 +28,8 @@ public AMSchedulerEventDeallocateContainer(ContainerId containerId, int schedule super(AMSchedulerEventType.S_CONTAINER_DEALLOCATE, schedulerId); this.containerId = containerId; } - + public ContainerId getContainerId() { return this.containerId; } } - diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventNodeBlacklistUpdate.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventNodeBlacklistUpdate.java index d22c0ec0ce..48ce7687b1 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventNodeBlacklistUpdate.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventNodeBlacklistUpdate.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java index 107fbf62c7..2a6b711724 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTAEnded.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -32,7 +32,7 @@ public class AMSchedulerEventTAEnded extends AMSchedulerEvent { private final String diagnostics; public AMSchedulerEventTAEnded(TaskAttempt attempt, ContainerId containerId, - TaskAttemptState state, TaskAttemptEndReason taskAttemptEndReason, String diagnostics, int schedulerId) { + TaskAttemptState state, TaskAttemptEndReason taskAttemptEndReason, String diagnostics, int schedulerId) { super(AMSchedulerEventType.S_TA_ENDED, schedulerId); this.attempt = attempt; this.containerId = containerId; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java index 0424c97eaf..1828090af0 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventTALaunchRequest.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -42,11 +42,11 @@ public class AMSchedulerEventTALaunchRequest extends AMSchedulerEvent { private final int taskCommId; public AMSchedulerEventTALaunchRequest(TezTaskAttemptID attemptId, - Resource capability, - TaskSpec remoteTaskSpec, TaskAttempt ta, - TaskLocationHint locationHint, int priority, - ContainerContext containerContext, - int schedulerId, int launcherId, int taskCommId) { + Resource capability, + TaskSpec remoteTaskSpec, TaskAttempt ta, + TaskLocationHint locationHint, int priority, + ContainerContext containerContext, + int schedulerId, int launcherId, int taskCommId) { super(AMSchedulerEventType.S_TA_LAUNCH_REQUEST, schedulerId); this.attemptId = attemptId; this.capability = capability; @@ -66,7 +66,7 @@ public TezTaskAttemptID getAttemptID() { public Resource getCapability() { return capability; } - + public TaskLocationHint getLocationHint() { return locationHint; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventType.java index cc52ef6bce..7e51697715 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventType.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/AMSchedulerEventType.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,7 +31,7 @@ public enum AMSchedulerEventType { S_NODE_HEALTHY, // The scheduler should have a way of knowing about unusable nodes. Acting on // this information to change requests etc is scheduler specific. - + // Producer : AMContainer S_CONTAINER_DEALLOCATE } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerAllocator.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerAllocator.java index 46c3dd8cdc..1088f47911 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerAllocator.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerAllocator.java @@ -1,27 +1,27 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.rm; import org.apache.hadoop.yarn.event.EventHandler; // TODO XXX Rename to AMScheduler. -public interface ContainerAllocator extends EventHandler{ +public interface ContainerAllocator extends EventHandler { // enum EventType { // @@ -29,5 +29,4 @@ public interface ContainerAllocator extends EventHandler{ // CONTAINER_DEALLOCATE, // CONTAINER_FAILED // } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherEvent.java index add3254ea0..43b35aea88 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherEventType.java index 079509c8fe..eee459863d 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherEventType.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherEventType.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherLaunchRequestEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherLaunchRequestEvent.java index 411451e3e0..72fa2d32a9 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherLaunchRequestEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherLaunchRequestEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,7 +31,7 @@ public ContainerLauncherLaunchRequestEvent(ContainerLaunchContext clc, Container container, int launcherId, int schedulerId, int taskCommId) { super(container.getId(), container.getNodeId(), container - .getContainerToken(), ContainerLauncherEventType.CONTAINER_LAUNCH_REQUEST, + .getContainerToken(), ContainerLauncherEventType.CONTAINER_LAUNCH_REQUEST, launcherId, schedulerId, taskCommId); this.clc = clc; this.container = container; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherStopRequestEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherStopRequestEvent.java index 69e7d30433..f5d8b4ba0f 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherStopRequestEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/ContainerLauncherStopRequestEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -30,5 +30,4 @@ public ContainerLauncherStopRequestEvent(ContainerId containerId, NodeId nodeId, super(containerId, nodeId, containerToken, ContainerLauncherEventType.CONTAINER_STOP_REQUEST, launcherId, schedulerId, taskCommId); } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/DagAwareYarnTaskScheduler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/DagAwareYarnTaskScheduler.java index 29f350f9d0..6bb2f7cbdf 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/DagAwareYarnTaskScheduler.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/DagAwareYarnTaskScheduler.java @@ -86,10 +86,10 @@ * A YARN task scheduler that is aware of the dependencies between vertices * in the DAG and takes them into account when deciding how to schedule * and preempt tasks. - * + *

    * This scheduler makes the assumption that vertex IDs start at 0 and are * densely allocated (i.e.: there are no "gaps" in the vertex ID space). - */ + */ public class DagAwareYarnTaskScheduler extends TaskScheduler implements AMRMClientAsync.CallbackHandler { private static final Logger LOG = LoggerFactory.getLogger(DagAwareYarnTaskScheduler.class); @@ -113,11 +113,15 @@ public class DagAwareYarnTaskScheduler extends TaskScheduler @GuardedBy("this") private final Map taskAssignments = new HashMap<>(); - /** A mapping from the vertex ID to the set of containers assigned to tasks for that vertex */ + /** + * A mapping from the vertex ID to the set of containers assigned to tasks for that vertex + */ @GuardedBy("this") private final Map> vertexAssignments = new HashMap<>(); - /** If vertex N has at least one task assigned to a container then the corresponding bit at index N is set */ + /** + * If vertex N has at least one task assigned to a container then the corresponding bit at index N is set + */ @GuardedBy("this") private final BitSet assignedVertices = new BitSet(); @@ -185,21 +189,21 @@ void initialize(AMRMClientAsyncWrapper client) throws Exception { TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED_DEFAULT); reuseNonLocal = conf - .getBoolean( - TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED, - TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED_DEFAULT); + .getBoolean( + TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED, + TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED_DEFAULT); Preconditions.checkArgument( - ((!reuseRackLocal && !reuseNonLocal) || (reuseRackLocal)), - "Re-use Rack-Local cannot be disabled if Re-use Non-Local has been" - + " enabled"); + ((!reuseRackLocal && !reuseNonLocal) || (reuseRackLocal)), + "Re-use Rack-Local cannot be disabled if Re-use Non-Local has been" + + " enabled"); reuseNewContainers = shouldReuseContainers && conf.getBoolean( TezConfiguration.TEZ_AM_CONTAINER_REUSE_NEW_CONTAINERS_ENABLED, TezConfiguration.TEZ_AM_CONTAINER_REUSE_NEW_CONTAINERS_ENABLED_DEFAULT); localitySchedulingDelay = conf.getLong( - TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, - TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS_DEFAULT); + TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, + TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS_DEFAULT); Preconditions.checkArgument(localitySchedulingDelay >= 0, "Locality Scheduling delay should be >=0"); @@ -207,7 +211,7 @@ void initialize(AMRMClientAsyncWrapper client) throws Exception { TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS, TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS_DEFAULT); Preconditions.checkArgument(idleContainerTimeoutMin >= 0 || idleContainerTimeoutMin == -1, - "Idle container release min timeout should be either -1 or >=0"); + "Idle container release min timeout should be either -1 or >=0"); idleContainerTimeoutMax = conf.getLong( TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS, @@ -215,7 +219,7 @@ void initialize(AMRMClientAsyncWrapper client) throws Exception { Preconditions.checkArgument( idleContainerTimeoutMax >= 0 && idleContainerTimeoutMax >= idleContainerTimeoutMin, "Idle container release max timeout should be >=0 and >= " + - TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS); + TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS); sessionNumMinHeldContainers = conf.getInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS_DEFAULT); @@ -235,19 +239,19 @@ void initialize(AMRMClientAsyncWrapper client) throws Exception { preemptionMaxWaitTime = conf.getInt(TezConfiguration.TEZ_AM_PREEMPTION_MAX_WAIT_TIME_MS, TezConfiguration.TEZ_AM_PREEMPTION_MAX_WAIT_TIME_MS_DEFAULT); - Preconditions.checkArgument(preemptionMaxWaitTime >=0, "Preemption max wait time must be >=0"); + Preconditions.checkArgument(preemptionMaxWaitTime >= 0, "Preemption max wait time must be >=0"); LOG.info("scheduler initialized with maxRMHeartbeatInterval:" + heartbeatIntervalMax + - " reuseEnabled:" + shouldReuseContainers + - " reuseRack:" + reuseRackLocal + - " reuseAny:" + reuseNonLocal + - " localityDelay:" + localitySchedulingDelay + - " preemptPercentage:" + preemptionPercentage + - " preemptMaxWaitTime:" + preemptionMaxWaitTime + - " numHeartbeatsBetweenPreemptions:" + numHeartbeatsBetweenPreemptions + - " idleContainerMinTimeout:" + idleContainerTimeoutMin + - " idleContainerMaxTimeout:" + idleContainerTimeoutMax + - " sessionMinHeldContainers:" + sessionNumMinHeldContainers); + " reuseEnabled:" + shouldReuseContainers + + " reuseRack:" + reuseRackLocal + + " reuseAny:" + reuseNonLocal + + " localityDelay:" + localitySchedulingDelay + + " preemptPercentage:" + preemptionPercentage + + " preemptMaxWaitTime:" + preemptionMaxWaitTime + + " numHeartbeatsBetweenPreemptions:" + numHeartbeatsBetweenPreemptions + + " idleContainerMinTimeout:" + idleContainerTimeoutMin + + " idleContainerMaxTimeout:" + idleContainerTimeoutMax + + " sessionMinHeldContainers:" + sessionNumMinHeldContainers); } @Override @@ -314,13 +318,13 @@ public void shutdown() throws Exception { } synchronized (this) { if (shouldUnregister && !hasUnregistered) { - TaskSchedulerContext.AppFinalStatus status = getContext().getFinalAppStatus(); - LOG.info("Unregistering from RM, exitStatus={} exitMessage={} trackingURL={}", - status.exitStatus, status.exitMessage, status.postCompletionTrackingUrl); - client.unregisterApplicationMaster(status.exitStatus, - status.exitMessage, - status.postCompletionTrackingUrl); - hasUnregistered = true; + TaskSchedulerContext.AppFinalStatus status = getContext().getFinalAppStatus(); + LOG.info("Unregistering from RM, exitStatus={} exitMessage={} trackingURL={}", + status.exitStatus, status.exitMessage, status.postCompletionTrackingUrl); + client.unregisterApplicationMaster(status.exitStatus, + status.exitMessage, + status.postCompletionTrackingUrl); + hasUnregistered = true; } } client.stop(); @@ -341,7 +345,7 @@ public void onContainersAllocated(List containers) { } private synchronized List assignNewContainers(List newContainers, - AMState appState, boolean isSession) { + AMState appState, boolean isSession) { // try to assign the containers as node-local List assignments = new ArrayList<>(newContainers.size()); List unassigned = new ArrayList<>(newContainers.size()); @@ -384,14 +388,14 @@ private synchronized List assignNewContainers(List newCon /** * Try to assign a newly acquired container to a task of the same priority. * - * @param hc the container to assign - * @param location the locality to consider for assignment + * @param hc the container to assign + * @param location the locality to consider for assignment * @param assignments list to update if container is assigned - * @param unassigned list to update if container is not assigned + * @param unassigned list to update if container is not assigned */ @GuardedBy("this") private void tryAssignNewContainer(HeldContainer hc, String location, - List assignments, List unassigned) { + List assignments, List unassigned) { List> results = client.getMatchingRequests(hc.getPriority(), location, hc.getCapability()); if (!results.isEmpty()) { @@ -414,38 +418,38 @@ private void tryAssignNewContainer(HeldContainer hc, String location, @GuardedBy("this") @Nullable private TaskRequest tryAssignReuseContainer(HeldContainer hc, - AMState appState, boolean isSession) { + AMState appState, boolean isSession) { if (stopRequested) { return null; } TaskRequest assignedRequest = null; switch (appState) { - case IDLE: - handleReuseContainerWhenIdle(hc, isSession); - break; - case RUNNING_APP: - if (requestTracker.isEmpty()) { - // treat no requests as if app is idle + case IDLE: handleReuseContainerWhenIdle(hc, isSession); - } else { - assignedRequest = tryAssignReuseContainerAppRunning(hc); - if (assignedRequest == null) { - if (hc.atMaxMatchLevel()) { - LOG.info("Releasing idle container {} due to pending requests", hc.getId()); - releaseContainer(hc); - } else { - hc.scheduleForReuse(localitySchedulingDelay); + break; + case RUNNING_APP: + if (requestTracker.isEmpty()) { + // treat no requests as if app is idle + handleReuseContainerWhenIdle(hc, isSession); + } else { + assignedRequest = tryAssignReuseContainerAppRunning(hc); + if (assignedRequest == null) { + if (hc.atMaxMatchLevel()) { + LOG.info("Releasing idle container {} due to pending requests", hc.getId()); + releaseContainer(hc); + } else { + hc.scheduleForReuse(localitySchedulingDelay); + } } } - } - break; - case COMPLETED: - LOG.info("Releasing container {} because app has completed", hc.getId()); - releaseContainer(hc); - break; - default: - throw new IllegalStateException("Unexpected app state " + appState); + break; + case COMPLETED: + LOG.info("Releasing container {} because app has completed", hc.getId()); + releaseContainer(hc); + break; + default: + throw new IllegalStateException("Unexpected app state " + appState); } return assignedRequest; @@ -485,7 +489,7 @@ private TaskRequest tryAssignReuseContainerAppRunning(HeldContainer hc) { return assignedRequest; } - for (Entry entry : requestTracker.getStatsEntries()) { + for (Entry entry : requestTracker.getStatsEntries()) { Priority priority = entry.getKey(); RequestPriorityStats stats = entry.getValue(); if (!stats.allowedVertices.intersects(stats.vertices)) { @@ -533,7 +537,7 @@ private TaskRequest tryAssignReuseContainerForAffinity(HeldContainer hc) { @GuardedBy("this") @Nullable private TaskRequest tryAssignReuseContainerForPriority(HeldContainer hc, String matchLocation, - Priority priority, BitSet allowedVertices) { + Priority priority, BitSet allowedVertices) { List> results = client.getMatchingRequests(priority, matchLocation, hc.getCapability()); if (results.isEmpty()) { return null; @@ -571,7 +575,7 @@ private void informAppAboutAssignments(List assignments) { * Inform the app about a task assignment. This should not be called with * any locks held. * - * @param request the corresponding task request + * @param request the corresponding task request * @param container the container assigned to the task */ private void informAppAboutAssignment(TaskRequest request, Container container) { @@ -600,7 +604,7 @@ private void informAppAboutAssignment(TaskRequest request, Container container) private void computeSessionContainers() { Map rackHeldNumber = new HashMap<>(); Map> nodeHeldContainers = new HashMap<>(); - for(HeldContainer heldContainer : heldContainers.values()) { + for (HeldContainer heldContainer : heldContainers.values()) { if (heldContainer.getSignature() == null) { // skip containers that have not been launched as there is no process to reuse continue; @@ -634,7 +638,7 @@ private void computeSessionContainers() { Entry entry = iter.next(); MutableInt rackCount = entry.getValue(); rackCount.decrement(); - if (rackCount.intValue() >=0) { + if (rackCount.intValue() >= 0) { containerCount++; rackToHoldNumber.get(entry.getKey()).increment(); } else { @@ -810,7 +814,7 @@ public synchronized void unblacklistNode(NodeId nodeId) { @Override public void allocateTask(Object task, Resource capability, String[] hosts, String[] racks, - Priority priority, Object containerSignature, Object clientCookie) { + Priority priority, Object containerSignature, Object clientCookie) { int vertexIndex = getContext().getVertexIndexForTask(task); TaskRequest request = new TaskRequest(task, vertexIndex, capability, hosts, racks, priority, containerSignature, clientCookie); @@ -819,7 +823,7 @@ public void allocateTask(Object task, Resource capability, String[] hosts, Strin @Override public void allocateTask(Object task, Resource capability, ContainerId containerId, - Priority priority, Object containerSignature, Object clientCookie) { + Priority priority, Object containerSignature, Object clientCookie) { String[] hosts = null; synchronized (this) { HeldContainer held = heldContainers.get(containerId); @@ -844,7 +848,7 @@ public void allocateTask(Object task, Resource capability, ContainerId container @Override public boolean deallocateTask(Object task, boolean taskSucceeded, - TaskAttemptEndReason endReason, String diagnostics) { + TaskAttemptEndReason endReason, String diagnostics) { ContainerId releasedLaunchedContainer = null; AMState appState = getContext().getAMState(); boolean isSession = getContext().isSession(); @@ -1084,7 +1088,7 @@ private HeldContainer tryAssignTaskToIdleContainer(TaskRequest request) { @GuardedBy("this") @Nullable private HeldContainer tryAssignTaskToIdleContainer(TaskRequest request, - List locations, EnumSet eligibleStates) { + List locations, EnumSet eligibleStates) { if (locations != null && !locations.isEmpty()) { for (String location : locations) { HeldContainer hc = tryAssignTaskToIdleContainer(request, location, eligibleStates); @@ -1099,7 +1103,7 @@ private HeldContainer tryAssignTaskToIdleContainer(TaskRequest request, @GuardedBy("this") @Nullable private HeldContainer tryAssignTaskToIdleContainer(TaskRequest request, - String location, EnumSet eligibleStates) { + String location, EnumSet eligibleStates) { Set containers = idleTracker.getByLocation(location); HeldContainer bestMatch = null; if (containers != null && !containers.isEmpty()) { @@ -1267,12 +1271,12 @@ static class TaskRequest extends AMRMClient.ContainerRequest { final ContainerId affinityContainerId; TaskRequest(Object task, int vertexIndex, Resource capability, String[] hosts, String[] racks, - Priority priority, Object signature, Object cookie) { - this(task, vertexIndex, capability, hosts, racks, priority, signature, cookie, null); + Priority priority, Object signature, Object cookie) { + this(task, vertexIndex, capability, hosts, racks, priority, signature, cookie, null); } TaskRequest(Object task, int vertexIndex, Resource capability, String[] hosts, String[] racks, - Priority priority, Object signature, Object cookie, ContainerId affinityContainerId) { + Priority priority, Object signature, Object cookie, ContainerId affinityContainerId) { super(capability, hosts, racks, priority); this.task = task; this.vertexIndex = vertexIndex; @@ -1343,11 +1347,15 @@ class HeldContainer implements Callable { @GuardedBy("DagAwareYarnTaskScheduler.this") HeldContainerState state = HeldContainerState.MATCHING_LOCAL; - /** The Future received when scheduling an idle container for re-allocation at a later time. */ + /** + * The Future received when scheduling an idle container for re-allocation at a later time. + */ @GuardedBy("DagAwareYarnTaskScheduler.this") Future future = null; - /** The collection of task requests that have specified this container as a scheduling affinity. */ + /** + * The collection of task requests that have specified this container as a scheduling affinity. + */ @GuardedBy("DagAwareYarnTaskScheduler.this") Collection affinities = null; @@ -1358,15 +1366,21 @@ class HeldContainer implements Callable { @GuardedBy("DagAwareYarnTaskScheduler.this") TaskRequest assignedRequest = null; - /** The task request corresponding to the last task that was assigned to this container. */ + /** + * The task request corresponding to the last task that was assigned to this container. + */ @GuardedBy("DagAwareYarnTaskScheduler.this") TaskRequest lastRequest = null; - /** The timestamp when the idle container will expire. 0 if the container is not idle. */ + /** + * The timestamp when the idle container will expire. 0 if the container is not idle. + */ @GuardedBy("DagAwareYarnTaskScheduler.this") long idleExpirationTimestamp = 0; - /** The timestamp when this container was assigned. 0 if the container is not assigned. */ + /** + * The timestamp when this container was assigned. 0 if the container is not assigned. + */ @GuardedBy("DagAwareYarnTaskScheduler.this") long assignmentTimestamp = 0; @@ -1490,46 +1504,46 @@ Object getLastTask() { String getMatchingLocation() { switch (state) { - case MATCHING_LOCAL: - return getHost(); - case MATCHING_RACK: - return getRack(); - case MATCHING_ANY: - return ResourceRequest.ANY; - default: - throw new IllegalStateException("Container " + getId() + " trying to match in state " + state); + case MATCHING_LOCAL: + return getHost(); + case MATCHING_RACK: + return getRack(); + case MATCHING_ANY: + return ResourceRequest.ANY; + default: + throw new IllegalStateException("Container " + getId() + " trying to match in state " + state); } } void moveToNextMatchingLevel() { switch (state) { - case MATCHING_LOCAL: - if (reuseRackLocal) { - state = HeldContainerState.MATCHING_RACK; - } - break; - case MATCHING_RACK: - if (reuseNonLocal) { - state = HeldContainerState.MATCHING_ANY; - } - break; - case MATCHING_ANY: - break; - default: - throw new IllegalStateException("Container " + getId() + " trying to match in state " + state); + case MATCHING_LOCAL: + if (reuseRackLocal) { + state = HeldContainerState.MATCHING_RACK; + } + break; + case MATCHING_RACK: + if (reuseNonLocal) { + state = HeldContainerState.MATCHING_ANY; + } + break; + case MATCHING_ANY: + break; + default: + throw new IllegalStateException("Container " + getId() + " trying to match in state " + state); } } boolean atMaxMatchLevel() { switch (state) { - case MATCHING_LOCAL: - return !reuseRackLocal; - case MATCHING_RACK: - return !reuseNonLocal; - case MATCHING_ANY: - return true; - default: - throw new IllegalStateException("Container " + getId() + " trying to match in state " + state); + case MATCHING_LOCAL: + return !reuseRackLocal; + case MATCHING_RACK: + return !reuseNonLocal; + case MATCHING_ANY: + return true; + default: + throw new IllegalStateException("Container " + getId() + " trying to match in state " + state); } } @@ -1543,7 +1557,7 @@ long getIdleExpirationTimestamp(long now) { if (idleExpirationTimestamp == 0) { if (idleContainerTimeoutMin > 0) { idleExpirationTimestamp = now + (idleContainerTimeoutMin == idleContainerTimeoutMax ? idleContainerTimeoutMin - : ThreadLocalRandom.current().nextLong(idleContainerTimeoutMin, idleContainerTimeoutMax)); + : ThreadLocalRandom.current().nextLong(idleContainerTimeoutMin, idleContainerTimeoutMax)); } else { idleExpirationTimestamp = Long.MAX_VALUE; } @@ -1647,7 +1661,9 @@ private static class TaskStatus { */ private class RequestTracker { private final Map requests = new HashMap<>(); - /** request map ordered by priority with highest priority first */ + /** + * request map ordered by priority with highest priority first + */ private final NavigableMap priorityStats = new TreeMap<>(Collections.reverseOrder()); private Priority highestPriority = null; @@ -1687,7 +1703,7 @@ TaskRequest remove(Object task) { private RequestPriorityStats addStatsForPriority(Priority priority) { BitSet allowedVerts = new BitSet(vertexDescendants.size()); - Entry lowerEntry = priorityStats.lowerEntry(priority); + Entry lowerEntry = priorityStats.lowerEntry(priority); if (lowerEntry != null) { // initialize the allowed vertices BitSet using the information derived // from the next higher priority entry @@ -1793,26 +1809,26 @@ private void decrVertexTaskCount(Priority priority, RequestPriorityStats stats, /** * Add a new vertex to a RequestPriorityStats. - * + *

    * Adding a vertex to the request stats requires updating the stats descendants bitmask to include the descendants * of the new vertex and also updating the allowedVertices bitmask for all lower priority requests to prevent any * task request from a descendant vertex in the DAG from being allocated. This avoids assigning allocations to * lower priority requests when a higher priority request of an ancestor is still pending, but it allows lower * priority requests to be satisfied if higher priority requests are not ancestors. This is particularly useful * for DAGs that have independent trees of vertices or significant, parallel branches within a tree. - * + *

    * Requests are blocked by taking the specified vertex's full descendant vertex bitmask in vertexDescendants and * clearing those bits for all lower priority requests. For the following example DAG where each vertex index * corresponds to its letter position (i.e.: A=0, B=1, C=2, etc.) - * - * A - * | - * C---B----E - * | | - * D F - * | - * G---H - * + *

    + * A + * | + * C---B----E + * | | + * D F + * | + * G---H + *

    * Vertices F, G, and H are descendants of E but all other vertices are not. The vertexDescendants bitmask for * vertex E is therefore 11100000b or 0xE0. When the first vertex E task request arrives we need to disallow * requests for all descendants of E. That is accomplished by iterating through the request stats for all lower @@ -1834,13 +1850,13 @@ private void addVertexToRequestStats(Priority priority, RequestPriorityStats sta /** * Removes a vertex from a RequestPriorityStats. - * + *

    * Removing a vertex is more expensive than adding a vertex. The stats contain bitmasks which only store on/off * values rather than reference counts. Therefore we must rebuild the descendants bitmasks from the remaining * vertices in the request stats. Once the new descendants mask is computed we then need to rebuild the * allowedVertices BitSet for all lower priority request stats in case the removal of this vertex unblocks lower * priority requests of a descendant vertex. - * + *

    * Rebuilding allowedVertices for the lower priorities involves starting with the allowedVertices mask at the * current priority then masking off the descendants at each priority level encountered, accumulating the results. * Any descendants of a level will be blocked at all lower levels. See the addVertexToRequestStats documentation @@ -1937,11 +1953,17 @@ BitSet createVertexBlockedSet() { * Tracks statistics on vertices that are requesting tasks at a particular priority */ private static class RequestPriorityStats { - /** Map from vertex ID to number of task requests for that vertex */ + /** + * Map from vertex ID to number of task requests for that vertex + */ final Map vertexTaskCount = new HashMap<>(); - /** BitSet of vertices that have oustanding requests at this priority */ + /** + * BitSet of vertices that have oustanding requests at this priority + */ final BitSet vertices; - /** BitSet of vertices that are descendants of this vertex */ + /** + * BitSet of vertices that are descendants of this vertex + */ final BitSet descendants; /** * BitSet of vertices that are allowed to be scheduled at this priority @@ -2018,6 +2040,7 @@ Set getByLocation(String location) { private interface ResourceCalculator { boolean anyAvailable(Resource rsrc); + void deductFrom(Resource total, Resource toSubtract); } @@ -2073,7 +2096,7 @@ protected void afterExecute(Runnable r, Throwable t) { try { ((Future) r).get(); } catch (ExecutionException ee) { - LOG.warn("Execution exception when running task in {}", Thread.currentThread().getName()); + LOG.warn("Execution exception when running task in {}", Thread.currentThread().getName()); t = ee.getCause(); } catch (InterruptedException ie) { LOG.warn("Thread ({}) interrupted: ", Thread.currentThread(), ie); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/LocalTaskSchedulerService.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/LocalTaskSchedulerService.java index cc213cb0cb..1765a90cd5 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/LocalTaskSchedulerService.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/LocalTaskSchedulerService.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.rm; @@ -90,9 +90,9 @@ public Resource getAvailableResources() { static Resource createResource(long runtimeMemory, int core) { if (runtimeMemory < 0 || core < 0) { throw new IllegalArgumentException("Negative Memory or Core provided!" - + "mem: "+runtimeMemory+" core:"+core); + + "mem: " + runtimeMemory + " core:" + core); } - return Resource.newInstance(Ints.checkedCast(runtimeMemory/(1024*1024)), core); + return Resource.newInstance(Ints.checkedCast(runtimeMemory / (1024 * 1024)), core); } @Override @@ -122,15 +122,15 @@ public void unblacklistNode(NodeId nodeId) { @Override public void allocateTask(Object task, Resource capability, String[] hosts, - String[] racks, Priority priority, Object containerSignature, - Object clientCookie) { + String[] racks, Priority priority, Object containerSignature, + Object clientCookie) { taskRequestHandler.addAllocateTaskRequest(task, capability, priority, clientCookie); } @Override public synchronized void allocateTask(Object task, Resource capability, - ContainerId containerId, Priority priority, Object containerSignature, - Object clientCookie) { + ContainerId containerId, Priority priority, Object containerSignature, + Object clientCookie) { // in local mode every task is already container level local taskRequestHandler.addAllocateTaskRequest(task, capability, priority, clientCookie); } @@ -250,7 +250,6 @@ public boolean equals(Object o) { public int hashCode() { return 7841 + (task != null ? task.hashCode() : 0); } - } static class AllocateTaskRequest extends TaskRequest implements Comparable { @@ -348,10 +347,10 @@ static class AsyncDelegateRequestHandler implements Runnable { final int MAX_TASKS; AsyncDelegateRequestHandler(LinkedBlockingQueue clientRequestQueue, - LocalContainerFactory localContainerFactory, - HashMap taskAllocations, - TaskSchedulerContext taskSchedulerContext, - Configuration conf) { + LocalContainerFactory localContainerFactory, + HashMap taskAllocations, + TaskSchedulerContext taskSchedulerContext, + Configuration conf) { this.clientRequestQueue = clientRequestQueue; this.localContainerFactory = localContainerFactory; this.taskAllocations = taskAllocations; @@ -366,6 +365,7 @@ void dagComplete() { vertexDescendants = null; } } + private void ensureVertexDescendants() { synchronized (descendantsLock) { if (vertexDescendants == null) { @@ -384,7 +384,7 @@ private void ensureVertexDescendants() { } public void addAllocateTaskRequest(Object task, Resource capability, Priority priority, - Object clientCookie) { + Object clientCookie) { try { int vertexIndex = taskSchedulerContext.getVertexIndexForTask(task); clientRequestQueue.put(new AllocateTaskRequest(task, vertexIndex, capability, priority, clientCookie)); @@ -432,18 +432,15 @@ void dispatchRequest() { try { SchedulerRequest request = clientRequestQueue.take(); if (request instanceof AllocateTaskRequest) { - taskRequestQueue.put((AllocateTaskRequest)request); + taskRequestQueue.put((AllocateTaskRequest) request); if (shouldPreempt()) { maybePreempt((AllocateTaskRequest) request); } - } - else if (request instanceof DeallocateTaskRequest) { - deallocateTask((DeallocateTaskRequest)request); - } - else if (request instanceof DeallocateContainerRequest) { - preemptTask((DeallocateContainerRequest)request); - } - else { + } else if (request instanceof DeallocateTaskRequest) { + deallocateTask((DeallocateTaskRequest) request); + } else if (request instanceof DeallocateContainerRequest) { + preemptTask((DeallocateContainerRequest) request); + } else { LOG.error("Unknown task request message: " + request); } } catch (InterruptedException e) { @@ -460,7 +457,7 @@ void maybePreempt(AllocateTaskRequest request) { Object task = entry.getKey(); ensureVertexDescendants(); if (vertexDescendants.get(request.vertexIndex).get(allocatedTask.request.vertexIndex)) { - LOG.info("Preempting task/container for task/priority:" + task + "/" + container + LOG.info("Preempting task/container for task/priority:" + task + "/" + container + " for " + request.task + "/" + priority); taskSchedulerContext.preemptContainer(allocatedTask.container.getId()); } @@ -484,8 +481,7 @@ void deallocateTask(DeallocateTaskRequest request) { AllocatedTask allocatedTask = taskAllocations.remove(request.task); if (allocatedTask != null) { taskSchedulerContext.containerBeingReleased(allocatedTask.container.getId()); - } - else { + } else { Iterator iter = taskRequestQueue.iterator(); while (iter.hasNext()) { TaskRequest taskRequest = iter.next(); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerContextImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerContextImpl.java index a31b4f1e2d..72805338cb 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerContextImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerContextImpl.java @@ -58,7 +58,6 @@ public TaskSchedulerContextImpl(TaskSchedulerManager taskSchedulerManager, AppCo this.appHostName = appHostname; this.clientPort = clientPort; this.initialUserPayload = initialUserPayload; - } // this may end up being called for a task+container pair that the app @@ -118,7 +117,6 @@ public UserPayload getInitialUserPayload() { return initialUserPayload; } - @Override public String getAppTrackingUrl() { return trackingUrl; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerContextImplWrapper.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerContextImplWrapper.java index 5e45e70993..4738d3aebb 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerContextImplWrapper.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerContextImplWrapper.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -53,7 +53,7 @@ class TaskSchedulerContextImplWrapper implements TaskSchedulerContext { private TaskSchedulerContext real; private ExecutorService executorService; - + /** * @param real the actual TaskSchedulerAppCallback * @param executorService the ExecutorService to be used to send these events. @@ -72,7 +72,7 @@ public void taskAllocated(Object task, Object appCookie, Container container) { @Override public void containerCompleted(Object taskLastAllocated, - ContainerStatus containerStatus) { + ContainerStatus containerStatus) { executorService.submit(new ContainerCompletedCallable(real, taskLastAllocated, containerStatus)); } @@ -95,7 +95,7 @@ public void appShutdownRequested() { @Override public void setApplicationRegistrationData(Resource maxContainerCapability, - Map appAcls, ByteBuffer key, String queueName) { + Map appAcls, ByteBuffer key, String queueName) { executorService.submit(new SetApplicationRegistrationDataCallable(real, maxContainerCapability, appAcls, key, queueName)); } @@ -116,7 +116,7 @@ public float getProgress() { throw new TezUncheckedException(e); } } - + @Override public void preemptContainer(ContainerId containerId) { executorService.submit(new PreemptContainerCallable(real, containerId)); @@ -195,7 +195,6 @@ public int getVertexIndexForTask(Object task) { // End of getters which do not need to go through a thread. Underlying implementation // does not use locks. - static abstract class TaskSchedulerContextCallbackBase { protected TaskSchedulerContext app; @@ -212,7 +211,7 @@ static class TaskAllocatedCallable extends TaskSchedulerContextCallbackBase private final Container container; public TaskAllocatedCallable(TaskSchedulerContext app, Object task, - Object appCookie, Container container) { + Object appCookie, Container container) { super(app); this.task = task; this.appCookie = appCookie; @@ -233,7 +232,7 @@ static class ContainerCompletedCallable extends TaskSchedulerContextCallbackBase private final ContainerStatus containerStatus; public ContainerCompletedCallable(TaskSchedulerContext app, - Object taskLastAllocated, ContainerStatus containerStatus) { + Object taskLastAllocated, ContainerStatus containerStatus) { super(app); this.taskLastAllocated = taskLastAllocated; this.containerStatus = containerStatus; @@ -251,7 +250,7 @@ static class ContainerBeingReleasedCallable extends private final ContainerId containerId; public ContainerBeingReleasedCallable(TaskSchedulerContext app, - ContainerId containerId) { + ContainerId containerId) { super(app); this.containerId = containerId; } @@ -268,7 +267,7 @@ static class NodesUpdatedCallable extends TaskSchedulerContextCallbackBase private final List updatedNodes; public NodesUpdatedCallable(TaskSchedulerContext app, - List updatedNodes) { + List updatedNodes) { super(app); this.updatedNodes = updatedNodes; } @@ -303,10 +302,10 @@ static class SetApplicationRegistrationDataCallable extends private final String queueName; public SetApplicationRegistrationDataCallable(TaskSchedulerContext app, - Resource maxContainerCapability, - Map appAcls, - ByteBuffer key, - String queueName) { + Resource maxContainerCapability, + Map appAcls, + ByteBuffer key, + String queueName) { super(app); this.maxContainerCapability = maxContainerCapability; this.appAcls = appAcls; @@ -346,19 +345,19 @@ public Void call() throws Exception { static class PreemptContainerCallable extends TaskSchedulerContextCallbackBase implements Callable { private final ContainerId containerId; - + public PreemptContainerCallable(TaskSchedulerContext app, ContainerId id) { super(app); this.containerId = id; } - + @Override public Void call() throws Exception { app.preemptContainer(containerId); return null; } } - + static class GetProgressCallable extends TaskSchedulerContextCallbackBase implements Callable { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java index 8a24ad3426..1207da47b3 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -95,9 +95,8 @@ import org.apache.tez.common.Preconditions; - public class TaskSchedulerManager extends AbstractService implements - EventHandler { + EventHandler { static final Logger LOG = LoggerFactory.getLogger(TaskSchedulerManager.class); static final String APPLICATION_ID_PLACEHOLDER = "__APPLICATION_ID__"; @@ -121,7 +120,7 @@ public class TaskSchedulerManager extends AbstractService implements private final WebUIService webUI; private final NamedEntityDescriptor[] taskSchedulerDescriptors; protected final TaskSchedulerWrapper[] taskSchedulers; - protected final ServicePluginLifecycleAbstractService []taskSchedulerServiceWrappers; + protected final ServicePluginLifecycleAbstractService[] taskSchedulerServiceWrappers; // Single executor service shared by all Schedulers for context callbacks @VisibleForTesting @@ -138,7 +137,7 @@ public class TaskSchedulerManager extends AbstractService implements private final HadoopShim hadoopShim; BlockingQueue eventQueue - = new LinkedBlockingQueue(); + = new LinkedBlockingQueue(); private final String yarnSchedulerClassName; @@ -226,7 +225,7 @@ public void setSignalled(boolean isSignalled) { public int getNumClusterNodes() { return cachedNodeCount; } - + public Resource getAvailableResources(int schedulerId) { try { return taskSchedulers[schedulerId].getAvailableResources(); @@ -267,41 +266,41 @@ private ExecutorService createAppCallbackExecutorService() { public synchronized void handleEvent(AMSchedulerEvent sEvent) { LOG.debug("Processing the event {}", sEvent); switch (sEvent.getType()) { - case S_TA_LAUNCH_REQUEST: - handleTaLaunchRequest((AMSchedulerEventTALaunchRequest) sEvent); - break; - case S_TA_STATE_UPDATED: - handleTAStateUpdated((AMSchedulerEventTAStateUpdated) sEvent); - break; - case S_TA_ENDED: // TaskAttempt considered complete. - AMSchedulerEventTAEnded event = (AMSchedulerEventTAEnded)sEvent; - switch(event.getState()) { - case FAILED: - case KILLED: - handleTAUnsuccessfulEnd(event); + case S_TA_LAUNCH_REQUEST: + handleTaLaunchRequest((AMSchedulerEventTALaunchRequest) sEvent); + break; + case S_TA_STATE_UPDATED: + handleTAStateUpdated((AMSchedulerEventTAStateUpdated) sEvent); + break; + case S_TA_ENDED: // TaskAttempt considered complete. + AMSchedulerEventTAEnded event = (AMSchedulerEventTAEnded) sEvent; + switch (event.getState()) { + case FAILED: + case KILLED: + handleTAUnsuccessfulEnd(event); + break; + case SUCCEEDED: + handleTASucceeded(event); + break; + default: + throw new TezUncheckedException("Unexpected TA_ENDED state: " + event.getState()); + } + break; + case S_CONTAINER_DEALLOCATE: + handleContainerDeallocate((AMSchedulerEventDeallocateContainer) sEvent); break; - case SUCCEEDED: - handleTASucceeded(event); + case S_NODE_UNBLACKLISTED: + // fall through + case S_NODE_BLACKLISTED: + handleNodeBlacklistUpdate((AMSchedulerEventNodeBlacklistUpdate) sEvent); + break; + case S_NODE_UNHEALTHY: + break; + case S_NODE_HEALTHY: + // Consider changing this to work like BLACKLISTING. break; default: - throw new TezUncheckedException("Unexpected TA_ENDED state: " + event.getState()); - } - break; - case S_CONTAINER_DEALLOCATE: - handleContainerDeallocate((AMSchedulerEventDeallocateContainer)sEvent); - break; - case S_NODE_UNBLACKLISTED: - // fall through - case S_NODE_BLACKLISTED: - handleNodeBlacklistUpdate((AMSchedulerEventNodeBlacklistUpdate)sEvent); - break; - case S_NODE_UNHEALTHY: - break; - case S_NODE_HEALTHY: - // Consider changing this to work like BLACKLISTING. - break; - default: - break; + break; } } @@ -355,7 +354,7 @@ private void handleNodeBlacklistUpdate(AMSchedulerEventNodeBlacklistUpdate event } private void handleContainerDeallocate( - AMSchedulerEventDeallocateContainer event) { + AMSchedulerEventDeallocateContainer event) { ContainerId containerId = event.getContainerId(); // TODO what happens to the task that was connected to this container? // current assumption is that it will eventually call handleTaStopRequest @@ -402,7 +401,7 @@ private void handleTAUnsuccessfulEnd(AMSchedulerEventTAEnded event) { // retroactive case ContainerId attemptContainerId = attempt.getAssignedContainerID(); - if(!wasContainerAllocated) { + if (!wasContainerAllocated) { LOG.info("Task: " + attempt.getTaskAttemptID() + " has no container assignment in the scheduler"); if (attemptContainerId != null) { @@ -453,7 +452,7 @@ private void handleTASucceeded(AMSchedulerEventTAEnded event) { try { wasContainerAllocated = taskSchedulers[event.getSchedulerId()].deallocateTask(attempt, - true, null, event.getDiagnostics()); + true, null, event.getDiagnostics()); } catch (Exception e) { String msg = "Error in TaskScheduler for handling Task De-allocation" + ", eventType=" + event.getType() @@ -484,10 +483,10 @@ private void handleTaLaunchRequest(AMSchedulerEventTALaunchRequest event) { Vertex vertex = appContext.getCurrentDAG().getVertex(taskAffinity.getVertexName()); Objects.requireNonNull(vertex, "Invalid vertex in task based affinity " + taskAffinity + " for attempt: " + taskAttempt.getTaskAttemptID()); - int taskIndex = taskAffinity.getTaskIndex(); - Preconditions.checkState(taskIndex >=0 && taskIndex < vertex.getTotalTasks(), - "Invalid taskIndex in task based affinity " + taskAffinity - + " for attempt: " + taskAttempt.getTaskAttemptID()); + int taskIndex = taskAffinity.getTaskIndex(); + Preconditions.checkState(taskIndex >= 0 && taskIndex < vertex.getTotalTasks(), + "Invalid taskIndex in task based affinity " + taskAffinity + + " for attempt: " + taskAttempt.getTaskAttemptID()); TaskAttempt affinityAttempt = vertex.getTask(taskIndex).getSuccessfulAttempt(); if (affinityAttempt != null) { Objects.requireNonNull(affinityAttempt.getAssignedContainerID(), @@ -592,8 +591,8 @@ TaskScheduler createYarnTaskScheduler(TaskSchedulerContext taskSchedulerContext, int schedulerId) throws TezException { LOG.info("Creating YARN TaskScheduler: {}", yarnSchedulerClassName); return ReflectionUtils.createClazzInstance(yarnSchedulerClassName, - new Class[] { TaskSchedulerContext.class }, - new Object[] { taskSchedulerContext }); + new Class[]{TaskSchedulerContext.class}, + new Object[]{taskSchedulerContext}); } @VisibleForTesting @@ -638,7 +637,6 @@ protected void instantiateSchedulers(String host, int port, String trackingUrl, } } - @Override public synchronized void serviceStart() throws Exception { // clientService is null in case of LocalDAGAppMaster @@ -652,7 +650,7 @@ public synchronized void serviceStart() throws Exception { String trackingUrl = (webUI != null) ? webUI.getTrackingURL() : ""; instantiateSchedulers(serviceAddr.getHostName(), serviceAddr.getPort(), trackingUrl, appContext); - for (int i = 0 ; i < taskSchedulers.length ; i++) { + for (int i = 0; i < taskSchedulers.length; i++) { taskSchedulerServiceWrappers[i].init(getConfig()); taskSchedulerServiceWrappers[i].start(); if (shouldUnregisterFlag.get()) { @@ -675,7 +673,7 @@ public void run() { } event = TaskSchedulerManager.this.eventQueue.take(); } catch (InterruptedException e) { - if(!stopEventHandling) { + if (!stopEventHandling) { LOG.warn("Continuing after interrupt : ", e); } continue; @@ -697,12 +695,12 @@ public void run() { }; this.eventHandlingThread.start(); } - + protected void notifyForTest() { } public void initiateStop() { - for (int i = 0 ; i < taskSchedulers.length ; i++) { + for (int i = 0; i < taskSchedulers.length; i++) { if (taskSchedulers[i] != null) { try { taskSchedulers[i].getTaskScheduler().initiateStop(); @@ -717,12 +715,12 @@ public void initiateStop() { @Override public void serviceStop() throws InterruptedException { - synchronized(this) { + synchronized (this) { this.stopEventHandling = true; if (eventHandlingThread != null) eventHandlingThread.interrupt(); } - for (int i = 0 ; i < taskSchedulers.length ; i++) { + for (int i = 0; i < taskSchedulers.length; i++) { if (taskSchedulers[i] != null) { taskSchedulerServiceWrappers[i].stop(); } @@ -734,8 +732,8 @@ public void serviceStop() throws InterruptedException { // TaskSchedulerAppCallback methods with schedulerId, where relevant public synchronized void taskAllocated(int schedulerId, Object task, - Object appCookie, - Container container) { + Object appCookie, + Container container) { AMSchedulerEventTALaunchRequest event = (AMSchedulerEventTALaunchRequest) appCookie; ContainerId containerId = container.getId(); @@ -747,7 +745,6 @@ public synchronized void taskAllocated(int schedulerId, Object task, .getNodeId(), schedulerId, container.getId())); } - TaskAttempt taskAttempt = event.getTaskAttempt(); // TODO - perhaps check if the task still needs this container // because the deallocateTask downcall may have raced with the @@ -765,7 +762,7 @@ public synchronized void taskAllocated(int schedulerId, Object task, } sendEvent(new AMContainerEventAssignTA(containerId, taskAttempt.getTaskAttemptID(), event.getRemoteTaskSpec(), event.getContainerContext().getLocalResources(), event - .getContainerContext().getCredentials(), event.getPriority())); + .getContainerContext().getCredentials(), event.getPriority())); } public synchronized void containerCompleted(int schedulerId, Object task, ContainerStatus containerStatus) { @@ -782,7 +779,7 @@ public synchronized void containerCompleted(int schedulerId, Object task, Contai } else if (exitStatus == ContainerExitStatus.DISKS_FAILED) { message = "Container disk failed. "; errCause = TaskAttemptTerminationCause.NODE_DISK_ERROR; - } else if (exitStatus != ContainerExitStatus.SUCCESS){ + } else if (exitStatus != ContainerExitStatus.SUCCESS) { message = "Container failed, exitCode=" + exitStatus + ". "; } if (containerStatus.getDiagnostics() != null) { @@ -819,7 +816,7 @@ public synchronized void appShutdownRequested(int schedulerId) { public synchronized void setApplicationRegistrationData( int schedulerId, Resource maxContainerCapability, - Map appAcls, + Map appAcls, ByteBuffer clientAMSecretKey, String queueName) { this.appContext.getClusterInfo().setMaxContainerCapability( @@ -855,7 +852,7 @@ public AppFinalStatus getFinalAppStatus() { finishState = hadoopShim.applyFinalApplicationStatusCorrection(finishState, dagAppMaster.isSession(), appMasterState == DAGAppMasterState.ERROR); List diagnostics = dagAppMaster.getDiagnostics(); - if(diagnostics != null) { + if (diagnostics != null) { for (String s : diagnostics) { sb.append(s).append("\n"); } @@ -868,8 +865,6 @@ public AppFinalStatus getFinalAppStatus() { return new AppFinalStatus(finishState, sb.toString(), historyUrl); } - - // Not synchronized to avoid deadlocks from TaskScheduler callbacks. // TaskScheduler uses a separate thread for it's callbacks. Since this method // returns a value which is required, the TaskScheduler wait for the call to @@ -935,7 +930,7 @@ public void reportError(int taskSchedulerIndex, ServicePluginError servicePlugin } public void dagCompleted() { - for (int i = 0 ; i < taskSchedulers.length ; i++) { + for (int i = 0; i < taskSchedulers.length; i++) { try { taskSchedulers[i].dagComplete(); } catch (Exception e) { @@ -987,7 +982,7 @@ public void preemptContainer(int schedulerId, ContainerId containerId) { public void setShouldUnregisterFlag() { LOG.info("TaskScheduler notified that it should unregister from RM"); this.shouldUnregisterFlag.set(true); - for (int i = 0 ; i < taskSchedulers.length ; i++) { + for (int i = 0; i < taskSchedulers.length; i++) { if (this.taskSchedulers[i] != null) { try { this.taskSchedulers[i].setShouldUnregister(); @@ -1045,7 +1040,7 @@ public String getHistoryUrl() { String historyUrl = ""; String historyUrlTemplate = config.get(TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE, - TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE_DEFAULT); + TezConfiguration.TEZ_AM_TEZ_UI_HISTORY_URL_TEMPLATE_DEFAULT); String historyUrlBase = config.get(TezConfiguration.TEZ_HISTORY_URL_BASE, ""); if (!historyUrlTemplate.isEmpty() && @@ -1077,5 +1072,4 @@ public String getHistoryUrl() { public String getTaskSchedulerClassName(int taskSchedulerIndex) { return taskSchedulers[taskSchedulerIndex].getTaskScheduler().getClass().getName(); } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java index 40bc3aa35c..1c9eee3782 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TezAMRMClientAsync.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -55,7 +55,7 @@ public LocalityRequestCounter() { } private TreeMap knownRequestsByPriority = - new TreeMap(); + new TreeMap(); public static TezAMRMClientAsync createAMRMClientAsync( int intervalMs, CallbackHandler callbackHandler) { @@ -72,29 +72,29 @@ public TezAMRMClientAsync( AMRMClientAsync.CallbackHandler callbackHandler) { super(client, intervalMs, callbackHandler); } - + public synchronized Priority getTopPriority() { if (knownRequestsByPriority.isEmpty()) { return null; } return knownRequestsByPriority.lastKey(); } - + // Remove after YARN-1723 is fixed public synchronized void addNodeToBlacklist(NodeId nodeId) { client.updateBlacklist(Collections.singletonList(nodeId.getHost()), null); } - + //Remove after YARN-1723 is fixed - public synchronized void removeNodeFromBlacklist(NodeId nodeId) { - client.updateBlacklist(null, Collections.singletonList(nodeId.getHost())); - } + public synchronized void removeNodeFromBlacklist(NodeId nodeId) { + client.updateBlacklist(null, Collections.singletonList(nodeId.getHost())); + } @Override public synchronized void addContainerRequest(T req) { super.addContainerRequest(req); boolean hasLocality = (req.getNodes() != null && !req.getNodes().isEmpty()) - || (req.getRacks() != null && !req.getRacks().isEmpty()); + || (req.getRacks() != null && !req.getRacks().isEmpty()); LocalityRequestCounter lrc = knownRequestsByPriority.get(req.getPriority()); if (lrc == null) { lrc = new LocalityRequestCounter(); @@ -111,9 +111,9 @@ public synchronized void addContainerRequest(T req) { public synchronized void removeContainerRequest(T req) { super.removeContainerRequest(req); boolean hasLocality = (req.getNodes() != null && !req.getNodes().isEmpty()) - || (req.getRacks() != null && !req.getRacks().isEmpty()); + || (req.getRacks() != null && !req.getRacks().isEmpty()); LocalityRequestCounter lrc = knownRequestsByPriority.get( - req.getPriority()); + req.getPriority()); if (hasLocality) { lrc.localityRequests.decrementAndGet(); } else { @@ -126,11 +126,11 @@ public synchronized void removeContainerRequest(T req) { } public synchronized List> - getMatchingRequestsForTopPriority( - String resourceName, Resource capability) { + getMatchingRequestsForTopPriority( + String resourceName, Resource capability) { // Sort based on reverse order. By default, Priority ordering is based on // highest numeric value being considered to be lowest priority. - Map.Entry entry = knownRequestsByPriority.lastEntry(); + Map.Entry entry = knownRequestsByPriority.lastEntry(); if (entry == null || entry.getValue() == null) { return Collections.emptyList(); } @@ -141,20 +141,19 @@ public synchronized void removeContainerRequest(T req) { // locality matching if (LOG.isDebugEnabled()) { LOG.debug("Over-ridding location request for matching containers as" - + " there are no pending requests that require locality at this" - + " priority" - + ", priority=" + p - + ", localityRequests=" + lrc.localityRequests - + ", noLocalityRequests=" + lrc.noLocalityRequests); + + " there are no pending requests that require locality at this" + + " priority" + + ", priority=" + p + + ", localityRequests=" + lrc.localityRequests + + ", noLocalityRequests=" + lrc.noLocalityRequests); } resourceName = ResourceRequest.ANY; } List> matched = - getMatchingRequests(p, resourceName, capability); + getMatchingRequests(p, resourceName, capability); if (matched != null && !matched.isEmpty()) { return matched; } return Collections.emptyList(); } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java index 6f51566f75..954ec97575 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerService.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.rm; @@ -84,11 +84,9 @@ } */ public class YarnTaskSchedulerService extends TaskScheduler - implements AMRMClientAsync.CallbackHandler { + implements AMRMClientAsync.CallbackHandler { private static final Logger LOG = LoggerFactory.getLogger(YarnTaskSchedulerService.class); - - final TezAMRMClientAsync amRmClient; final ContainerSignatureMatcher containerSignatureMatcher; @@ -99,36 +97,36 @@ public class YarnTaskSchedulerService extends TaskScheduler // type is linked hash map to maintain order of incoming requests Map taskRequests = - new LinkedHashMap(); + new LinkedHashMap(); // LinkedHashMap is need in getProgress() LinkedHashMap taskAllocations = - new LinkedHashMap(); + new LinkedHashMap(); /** * Tracks last task assigned to a known container. */ Map containerAssignments = - new HashMap(); + new HashMap(); // Remove inUse depending on resolution of TEZ-1129 - Set inUseContainers = Sets.newHashSet(); + Set inUseContainers = Sets.newHashSet(); HashMap releasedContainers = - new HashMap(); + new HashMap(); /** * Map of containers currently being held by the TaskScheduler. */ Map heldContainers = new HashMap(); - + Set priorityHasAffinity = Sets.newHashSet(); Set blacklistedNodes = Collections .newSetFromMap(new ConcurrentHashMap()); - + Resource totalResources = Resource.newInstance(0, 0); Resource allocatedResources = Resource.newInstance(0, 0); long numHeartbeats = 0; long heartbeatAtLastPreemption = 0; int numHeartbeatsBetweenPreemptions = 0; - + final String appHostName; final int appHostPort; final String appTrackingUrl; @@ -147,12 +145,12 @@ public class YarnTaskSchedulerService extends TaskScheduler int sessionNumMinHeldContainers = 0; int preemptionPercentage = 0; long preemptionMaxWaitTime = 0; - + long highestWaitingRequestWaitStartTime = 0; Priority highestWaitingRequestPriority = null; - + Set sessionMinHeldContainers = Sets.newHashSet(); - + private final Configuration conf; @VisibleForTesting @@ -163,7 +161,7 @@ static class CRCookie { private Object task; private Object appCookie; private Object containerSignature; - + CRCookie(Object task, Object appCookie, Object containerSignature) { this.task = task; this.appCookie = appCookie; @@ -177,7 +175,7 @@ Object getTask() { Object getAppCookie() { return appCookie; } - + Object getContainerSignature() { return containerSignature; } @@ -211,7 +209,7 @@ public CookieContainerRequest( CRCookie getCookie() { return cookie; } - + ContainerId getAffinitizedContainer() { return affinitizedContainerId; } @@ -236,7 +234,7 @@ public YarnTaskSchedulerService(TaskSchedulerContext taskSchedulerContext) { @Private @VisibleForTesting YarnTaskSchedulerService(TaskSchedulerContext taskSchedulerContext, - TezAMRMClientAsync client) { + TezAMRMClientAsync client) { super(taskSchedulerContext); this.containerSignatureMatcher = taskSchedulerContext.getContainerSignatureMatcher(); this.amRmClient = client; @@ -291,17 +289,17 @@ public synchronized void initialize() { TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED_DEFAULT); reuseNonLocal = conf - .getBoolean( - TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED, - TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED_DEFAULT); + .getBoolean( + TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED, + TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED_DEFAULT); Preconditions.checkArgument( - ((!reuseRackLocal && !reuseNonLocal) || (reuseRackLocal)), - "Re-use Rack-Local cannot be disabled if Re-use Non-Local has been" - + " enabled"); + ((!reuseRackLocal && !reuseNonLocal) || (reuseRackLocal)), + "Re-use Rack-Local cannot be disabled if Re-use Non-Local has been" + + " enabled"); localitySchedulingDelay = conf.getLong( - TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, - TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS_DEFAULT); + TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, + TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS_DEFAULT); Preconditions.checkArgument(localitySchedulingDelay >= 0, "Locality Scheduling delay should be >=0"); @@ -309,52 +307,52 @@ public synchronized void initialize() { TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS, TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS_DEFAULT); Preconditions.checkArgument(idleContainerTimeoutMin >= 0 || idleContainerTimeoutMin == -1, - "Idle container release min timeout should be either -1 or >=0"); - + "Idle container release min timeout should be either -1 or >=0"); + idleContainerTimeoutMax = conf.getLong( TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS, TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS_DEFAULT); Preconditions.checkArgument( idleContainerTimeoutMax >= 0 && idleContainerTimeoutMax >= idleContainerTimeoutMin, - "Idle container release max timeout should be >=0 and >= " + - TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS); - - sessionNumMinHeldContainers = conf.getInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, + "Idle container release max timeout should be >=0 and >= " + + TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS); + + sessionNumMinHeldContainers = conf.getInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS_DEFAULT); - Preconditions.checkArgument(sessionNumMinHeldContainers >= 0, + Preconditions.checkArgument(sessionNumMinHeldContainers >= 0, "Session minimum held containers should be >=0"); - - preemptionPercentage = conf.getInt(TezConfiguration.TEZ_AM_PREEMPTION_PERCENTAGE, + + preemptionPercentage = conf.getInt(TezConfiguration.TEZ_AM_PREEMPTION_PERCENTAGE, TezConfiguration.TEZ_AM_PREEMPTION_PERCENTAGE_DEFAULT); Preconditions.checkArgument(preemptionPercentage >= 0 && preemptionPercentage <= 100, "Preemption percentage should be between 0-100"); - + numHeartbeatsBetweenPreemptions = conf.getInt( TezConfiguration.TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS, TezConfiguration.TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS_DEFAULT); - Preconditions.checkArgument(numHeartbeatsBetweenPreemptions >= 1, + Preconditions.checkArgument(numHeartbeatsBetweenPreemptions >= 1, "Heartbeats between preemptions should be >=1"); - - preemptionMaxWaitTime = conf.getInt(TezConfiguration.TEZ_AM_PREEMPTION_MAX_WAIT_TIME_MS, + + preemptionMaxWaitTime = conf.getInt(TezConfiguration.TEZ_AM_PREEMPTION_MAX_WAIT_TIME_MS, TezConfiguration.TEZ_AM_PREEMPTION_MAX_WAIT_TIME_MS_DEFAULT); - Preconditions.checkArgument(preemptionMaxWaitTime >=0, "Preemption max wait time must be >=0"); + Preconditions.checkArgument(preemptionMaxWaitTime >= 0, "Preemption max wait time must be >=0"); delayedContainerManager = new DelayedContainerManager(); NODE_LOCAL_ASSIGNER = new NodeLocalContainerAssigner(); RACK_LOCAL_ASSIGNER = new RackLocalContainerAssigner(); NON_LOCAL_ASSIGNER = new NonLocalContainerAssigner(); LOG.info("YarnTaskScheduler initialized with configuration: " + - "maxRMHeartbeatInterval: " + heartbeatIntervalMax + - ", containerReuseEnabled: " + shouldReuseContainers + - ", reuseRackLocal: " + reuseRackLocal + - ", reuseNonLocal: " + reuseNonLocal + - ", localitySchedulingDelay: " + localitySchedulingDelay + - ", preemptionPercentage: " + preemptionPercentage + - ", preemptionMaxWaitTime: " + preemptionMaxWaitTime + - ", numHeartbeatsBetweenPreemptions: " + numHeartbeatsBetweenPreemptions + - ", idleContainerMinTimeout: " + idleContainerTimeoutMin + - ", idleContainerMaxTimeout: " + idleContainerTimeoutMax + - ", sessionMinHeldContainers: " + sessionNumMinHeldContainers); + "maxRMHeartbeatInterval: " + heartbeatIntervalMax + + ", containerReuseEnabled: " + shouldReuseContainers + + ", reuseRackLocal: " + reuseRackLocal + + ", reuseNonLocal: " + reuseNonLocal + + ", localitySchedulingDelay: " + localitySchedulingDelay + + ", preemptionPercentage: " + preemptionPercentage + + ", preemptionMaxWaitTime: " + preemptionMaxWaitTime + + ", numHeartbeatsBetweenPreemptions: " + numHeartbeatsBetweenPreemptions + + ", idleContainerMinTimeout: " + idleContainerTimeoutMin + + ", idleContainerMaxTimeout: " + idleContainerTimeoutMax + + ", sessionMinHeldContainers: " + sessionNumMinHeldContainers); } @Override @@ -364,8 +362,8 @@ public void start() { synchronized (this) { amRmClient.start(); response = amRmClient.registerApplicationMaster(appHostName, - appHostPort, - appTrackingUrl); + appHostPort, + appTrackingUrl); } // upcall to app outside locks getContext().setApplicationRegistrationData( @@ -432,14 +430,14 @@ public void onContainersCompleted(List statuses) { return; } Map appContainerStatus = - new HashMap(statuses.size()); + new HashMap(statuses.size()); synchronized (this) { - for(ContainerStatus containerStatus : statuses) { + for (ContainerStatus containerStatus : statuses) { ContainerId completedId = containerStatus.getContainerId(); HeldContainer delayedContainer = heldContainers.get(completedId); Object task = releasedContainers.remove(completedId); - if(task != null){ + if (task != null) { if (delayedContainer != null) { LOG.warn("Held container should be null since releasedContainer is not"); } @@ -449,7 +447,7 @@ public void onContainersCompleted(List statuses) { // completion of a container we had released earlier // an allocated container completed. notify app LOG.debug("Released container completed:{} last allocated to task: {}", - completedId, task); + completedId, task); appContainerStatus.put(task, containerStatus); continue; } @@ -463,7 +461,7 @@ public void onContainersCompleted(List statuses) { } else { LOG.warn("Held container expected to be not null for a non-AM-released container"); } - if(task != null) { + if (task != null) { // completion of a container we have allocated currently // an allocated container completed. notify app. This will cause attempt to get killed LOG.info( @@ -491,7 +489,7 @@ public void onContainersCompleted(List statuses) { @Override public void onContainersAllocated(List containers) { if (isStopStarted.get()) { - LOG.info("Ignoring container allocations because application is shutting down. Num " + + LOG.info("Ignoring container allocations because application is shutting down. Num " + containers.size()); if (LOG.isDebugEnabled()) { for (Container container : containers) { @@ -505,7 +503,7 @@ public void onContainersAllocated(List containers) { if (LOG.isDebugEnabled()) { StringBuilder sb = new StringBuilder(); - for (Container container: containers) { + for (Container container : containers) { sb.append(container.getId()).append(", "); } LOG.debug("Assigned New Containers: " + sb.toString()); @@ -540,7 +538,7 @@ public void onContainersAllocated(List containers) { * @return Assignments. */ private synchronized Map - assignNewlyAllocatedContainers(Iterable containers) { + assignNewlyAllocatedContainers(Iterable containers) { boolean amInCompletionState = (getContext().getAMState() == AMState.COMPLETED); Map assignedContainers = @@ -565,11 +563,11 @@ public void onContainersAllocated(List containers) { } private synchronized Map - tryAssignReUsedContainers(Iterable containers) { + tryAssignReUsedContainers(Iterable containers) { boolean amInCompletionState = (getContext().getAMState() == AMState.COMPLETED); Map assignedContainers = - new HashMap(); + new HashMap(); // Honor locality and match as many as possible if (!amInCompletionState) { @@ -583,7 +581,7 @@ public void onContainersAllocated(List containers) { return assignedContainers; } - + @VisibleForTesting long getHeldContainerExpireTime(long startTime) { // expire time is at least extended by min time. @@ -594,7 +592,7 @@ long getHeldContainerExpireTime(long startTime) { long expireTimeMax = startTime + idleContainerTimeoutMax; expireTime = ThreadLocalRandom.current().nextLong(expireTime, expireTimeMax); } - + return expireTime; } @@ -605,7 +603,7 @@ long getHeldContainerExpireTime(long startTime) { */ private synchronized Map - assignDelayedContainer(HeldContainer heldContainer) { + assignDelayedContainer(HeldContainer heldContainer) { AMState state = getContext().getAMState(); @@ -634,7 +632,7 @@ long getHeldContainerExpireTime(long startTime) { // session mode and need to hold onto containers and not done so already determineMinHeldContainers(); } - + heldContainer.resetLocalityMatchLevel(); long currentTime = System.currentTimeMillis(); boolean releaseContainer = false; @@ -652,10 +650,10 @@ long getHeldContainerExpireTime(long startTime) { // the rest of the code. heldContainer.setContainerExpiryTime(getHeldContainerExpireTime(currentTime)); } else { - releaseContainer = true; + releaseContainer = true; } } - + if (releaseContainer) { LOG.info("No taskRequests. Container's idle timeout delay expired or is new. " + "Releasing container" @@ -667,8 +665,8 @@ long getHeldContainerExpireTime(long startTime) { + ", heldContainers=" + heldContainers.size() + ", delayedContainers=" + delayedContainerManager.delayedContainers.size() + ", isNew=" + isNew); - releaseUnassignedContainers( - Collections.singletonList((heldContainer.getContainer()))); + releaseUnassignedContainers( + Collections.singletonList((heldContainer.getContainer()))); } else { // no outstanding work and container idle timeout not expired if (LOG.isDebugEnabled()) { @@ -701,9 +699,9 @@ long getHeldContainerExpireTime(long startTime) { sessionMinHeldContainers.clear(); } HeldContainer.LocalityMatchLevel localityMatchLevel = - heldContainer.getLocalityMatchLevel(); + heldContainer.getLocalityMatchLevel(); Map assignedContainers = - new HashMap(); + new HashMap(); Container containerToAssign = heldContainer.container; @@ -722,7 +720,7 @@ long getHeldContainerExpireTime(long startTime) { NODE_LOCAL_ASSIGNER, assignedContainers, true); if (LOG.isDebugEnabled() && assignedContainers.isEmpty()) { LOG.debug("Failed to assign tasks to delayed container using node" - + ", containerId=" + heldContainer.getContainer().getId()); + + ", containerId=" + heldContainer.getContainer().getId()); } } @@ -731,14 +729,14 @@ long getHeldContainerExpireTime(long startTime) { // if scheduling delay is 0, match at RACK allowed without a sleep if (assignedContainers.isEmpty()) { if ((reuseRackLocal || isNew) && (localitySchedulingDelay == 0 || - (localityMatchLevel.equals(HeldContainer.LocalityMatchLevel.RACK) - || localityMatchLevel.equals( - HeldContainer.LocalityMatchLevel.NON_LOCAL)))) { + (localityMatchLevel.equals(HeldContainer.LocalityMatchLevel.RACK) + || localityMatchLevel.equals( + HeldContainer.LocalityMatchLevel.NON_LOCAL)))) { assignReUsedContainerWithLocation(containerToAssign, RACK_LOCAL_ASSIGNER, assignedContainers, false); if (LOG.isDebugEnabled() && assignedContainers.isEmpty()) { LOG.debug("Failed to assign tasks to delayed container using rack" - + ", containerId=" + heldContainer.getContainer().getId()); + + ", containerId=" + heldContainer.getContainer().getId()); } } } @@ -749,8 +747,8 @@ long getHeldContainerExpireTime(long startTime) { if (assignedContainers.isEmpty()) { if ((reuseNonLocal || isNew) && (localitySchedulingDelay == 0 || localityMatchLevel.equals( - HeldContainer.LocalityMatchLevel.NON_LOCAL))) { - assignReUsedContainerWithLocation(containerToAssign, + HeldContainer.LocalityMatchLevel.NON_LOCAL))) { + assignReUsedContainerWithLocation(containerToAssign, NON_LOCAL_ASSIGNER, assignedContainers, false); if (LOG.isDebugEnabled() && assignedContainers.isEmpty()) { LOG.debug("Failed to assign tasks to delayed container using non-local" @@ -773,28 +771,28 @@ long getHeldContainerExpireTime(long startTime) { // we cannot avoid releasing containers. Or else we may not be able to // get new containers from YARN to match the pending request if (!isNew && heldContainer.getContainerExpiryTime() <= currentTime - && idleContainerTimeoutMin != -1) { + && idleContainerTimeoutMin != -1) { LOG.info("Container's idle timeout expired. Releasing container" + ", containerId=" + heldContainer.container.getId() + ", containerExpiryTime=" + heldContainer.getContainerExpiryTime() + ", idleTimeoutMin=" + idleContainerTimeoutMin); releaseUnassignedContainers( - Lists.newArrayList(heldContainer.container)); + Lists.newArrayList(heldContainer.container)); } else { // Let's decide if this container has hit the end of the road // EOL true if container's match level is NON-LOCAL boolean hitFinalMatchLevel = localityMatchLevel.equals( - HeldContainer.LocalityMatchLevel.NON_LOCAL); + HeldContainer.LocalityMatchLevel.NON_LOCAL); if (!hitFinalMatchLevel) { // EOL also true if locality delay is 0 // or rack-local or non-local is disabled heldContainer.incrementLocalityMatchLevel(); if (localitySchedulingDelay == 0 || (!reuseRackLocal - || (!reuseNonLocal && + || (!reuseNonLocal && heldContainer.getLocalityMatchLevel().equals( HeldContainer.LocalityMatchLevel.NON_LOCAL)))) { hitFinalMatchLevel = true; @@ -806,7 +804,7 @@ long getHeldContainerExpireTime(long startTime) { hitFinalMatchLevel = false; } } - + if (hitFinalMatchLevel) { boolean safeToRelease = true; Priority topPendingPriority = amRmClient.getTopPriority(); @@ -819,10 +817,10 @@ long getHeldContainerExpireTime(long startTime) { // to give this container to us again safeToRelease = false; } - + // Are there any pending requests at any priority? // release if there are tasks or this is not a session - if (safeToRelease && + if (safeToRelease && (!taskRequests.isEmpty() || !getContext().isSession())) { LOG.info("Releasing held container as either there are pending but " + " unmatched requests or this is not a session" @@ -831,13 +829,13 @@ long getHeldContainerExpireTime(long startTime) { + ", isSession=" + getContext().isSession() + ". isNew=" + isNew); releaseUnassignedContainers( - Lists.newArrayList(heldContainer.container)); + Lists.newArrayList(heldContainer.container)); } else { // if no tasks, treat this like an idle session heldContainer.resetLocalityMatchLevel(); delayedContainerManager.addDelayedContainer( - heldContainer.getContainer(), - currentTime + localitySchedulingDelay); + heldContainer.getContainer(), + currentTime + localitySchedulingDelay); } } else { // Schedule delay container to match at a later try @@ -855,8 +853,8 @@ long getHeldContainerExpireTime(long startTime) { } else { // ignore all other cases? LOG.warn("Received a request to assign re-used containers when AM was " - + " in state: " + state + ". Ignoring request and releasing container" - + ": " + heldContainer.getContainer().getId()); + + " in state: " + state + ". Ignoring request and releasing container" + + ": " + heldContainer.getContainer().getId()); releaseUnassignedContainers(Lists.newArrayList(heldContainer.container)); } @@ -868,7 +866,7 @@ public synchronized void dagComplete() { for (HeldContainer heldContainer : heldContainers.values()) { heldContainer.resetLocalityMatchLevel(); } - synchronized(delayedContainerManager) { + synchronized (delayedContainerManager) { delayedContainerManager.notify(); } } @@ -898,7 +896,7 @@ public float getProgress() { return 1; } - if(totalResources.getMemory() == 0) { + if (totalResources.getMemory() == 0) { // assume this is the first allocate callback. nothing is allocated. // available resource = totalResource // TODO this will not handle dynamic changes in resources @@ -940,7 +938,7 @@ public synchronized void blacklistNode(NodeId nodeId) { amRmClient.addNodeToBlacklist(nodeId); blacklistedNodes.add(nodeId); } - + @Override public synchronized void unblacklistNode(NodeId nodeId) { if (blacklistedNodes.remove(nodeId)) { @@ -948,7 +946,7 @@ public synchronized void unblacklistNode(NodeId nodeId) { amRmClient.removeNodeFromBlacklist(nodeId); } } - + @Override public synchronized void allocateTask( Object task, @@ -964,11 +962,11 @@ public synchronized void allocateTask( // TODO extra memory allocation CRCookie cookie = new CRCookie(task, clientCookie, containerSignature); CookieContainerRequest request = new CookieContainerRequest( - capability, hosts, racks, priority, cookie); + capability, hosts, racks, priority, cookie); addRequestAndTrigger(task, request, hosts, racks); } - + @Override public synchronized void allocateTask( Object task, @@ -990,29 +988,29 @@ public synchronized void allocateTask( priorityHasAffinity.add(priority); } else { LOG.warn("Matching requested to container: " + containerId + - " but requested capability: " + capability + - " does not fit in container resource: " + container.getResource()); + " but requested capability: " + capability + + " does not fit in container resource: " + container.getResource()); } } else { LOG.warn("Matching requested to unknown container: " + containerId); } - + CRCookie cookie = new CRCookie(task, clientCookie, containerSignature); CookieContainerRequest request = new CookieContainerRequest( - capability, containerId, hosts, racks, priority, cookie); + capability, containerId, hosts, racks, priority, cookie); addRequestAndTrigger(task, request, hosts, racks); } - + private void addRequestAndTrigger(Object task, CookieContainerRequest request, - String[] hosts, String[] racks) { + String[] hosts, String[] racks) { addTaskRequest(task, request); // See if any of the delayedContainers can be used for this task. delayedContainerManager.triggerScheduling(true); LOG.info("Allocation request for task: " + task + - " with request: " + request + - " host: " + ((hosts != null && hosts.length > 0) ? hosts[0] : "null") + - " rack: " + ((racks != null && racks.length > 0) ? racks[0] : "null")); + " with request: " + request + + " host: " + ((hosts != null && hosts.length > 0) ? hosts[0] : "null") + + " rack: " + ((racks != null && racks.length > 0) ? racks[0] : "null")); } /** @@ -1082,11 +1080,11 @@ public boolean deallocateTask(Object task, boolean taskSucceeded, } return true; } - + @Override public synchronized Object deallocateContainer(ContainerId containerId) { Object task = unAssignContainer(containerId, true); - if(task != null) { + if (task != null) { // non-standard case for the app layer to deallocate container LOG.info("Deallocated container: " + containerId + " from task: " + task); @@ -1130,34 +1128,34 @@ boolean canFit(Resource arg0, Resource arg1) { int mem1 = arg1.getMemory(); int cpu0 = arg0.getVirtualCores(); int cpu1 = arg1.getVirtualCores(); - - if(mem0 <= mem1 && cpu0 <= cpu1) { + + if (mem0 <= mem1 && cpu0 <= cpu1) { return true; } - return false; + return false; } static int scaleDownByPreemptionPercentage(int original, int percent) { - return (int) Math.ceil((original * percent)/100.f); + return (int) Math.ceil((original * percent) / 100.f); } - + private String constructPreemptionPeriodicLog(Resource freeResource) { return "Allocated: " + allocatedResources + - " Free: " + freeResource + - " pendingRequests: " + taskRequests.size() + - " delayedContainers: " + delayedContainerManager.delayedContainers.size() + - " heartbeats: " + numHeartbeats + - " lastPreemptionHeartbeat: " + heartbeatAtLastPreemption + - ((highestWaitingRequestPriority != null) ? - (" highestWaitingRequestWaitStartTime: " + highestWaitingRequestWaitStartTime + - " highestWaitingRequestPriority: " + highestWaitingRequestPriority.toString()) : ""); - } - + " Free: " + freeResource + + " pendingRequests: " + taskRequests.size() + + " delayedContainers: " + delayedContainerManager.delayedContainers.size() + + " heartbeats: " + numHeartbeats + + " lastPreemptionHeartbeat: " + heartbeatAtLastPreemption + + ((highestWaitingRequestPriority != null) ? + (" highestWaitingRequestWaitStartTime: " + highestWaitingRequestWaitStartTime + + " highestWaitingRequestPriority: " + highestWaitingRequestPriority.toString()) : ""); + } + private void resetHighestWaitingPriority(Priority newPri) { highestWaitingRequestPriority = newPri; highestWaitingRequestWaitStartTime = 0; } - + boolean preemptIfNeeded() { if (preemptionPercentage == 0) { // turned off @@ -1175,43 +1173,43 @@ boolean preemptIfNeeded() { } } assert freeResources.getMemory() >= 0; - + CookieContainerRequest highestPriRequest = null; int numHighestPriRequests = 0; - for(CookieContainerRequest request : taskRequests.values()) { - if(highestPriRequest == null) { + for (CookieContainerRequest request : taskRequests.values()) { + if (highestPriRequest == null) { highestPriRequest = request; numHighestPriRequests = 1; - } else if(isHigherPriority(request.getPriority(), - highestPriRequest.getPriority())){ + } else if (isHigherPriority(request.getPriority(), + highestPriRequest.getPriority())) { highestPriRequest = request; numHighestPriRequests = 1; } else if (request.getPriority().equals(highestPriRequest.getPriority())) { numHighestPriRequests++; } } - + if (highestPriRequest == null) { // nothing pending resetHighestWaitingPriority(null); return true; } - + // reset the wait time when waiting priority changes to prevent carry over of the value if (highestWaitingRequestPriority == null || !highestPriRequest.getPriority().equals(highestWaitingRequestPriority)) { resetHighestWaitingPriority(highestPriRequest.getPriority()); } - + long currTime = System.currentTimeMillis(); if (highestWaitingRequestWaitStartTime == 0) { highestWaitingRequestWaitStartTime = currTime; } - boolean preemptionWaitDeadlineCrossed = + boolean preemptionWaitDeadlineCrossed = (currTime - highestWaitingRequestWaitStartTime) > preemptionMaxWaitTime ? true : false; - if(!preemptionWaitDeadlineCrossed && + if (!preemptionWaitDeadlineCrossed && fitsIn(highestPriRequest.getCapability(), freeResources)) { LOG.debug("{} fits in free resources", highestPriRequest); if (numHeartbeats % 50 == 1) { @@ -1219,7 +1217,7 @@ boolean preemptIfNeeded() { } return true; } - + if (preemptionWaitDeadlineCrossed) { // check if anything lower priority is running - priority inversion // this check could have been done earlier but in the common case @@ -1227,12 +1225,12 @@ boolean preemptIfNeeded() { // in the normal case without priority inversion. So do this expensive // iteration now boolean lowerPriRunning = false; - for(Map.Entry entry : taskAllocations.entrySet()) { + for (Map.Entry entry : taskAllocations.entrySet()) { HeldContainer heldContainer = heldContainers.get(entry.getValue().getId()); CookieContainerRequest lastTaskInfo = heldContainer.getLastTaskInfo(); Priority taskPriority = lastTaskInfo.getPriority(); Object signature = lastTaskInfo.getCookie().getContainerSignature(); - if(isHigherPriority(highestPriRequest.getPriority(), taskPriority)) { + if (isHigherPriority(highestPriRequest.getPriority(), taskPriority)) { // lower priority task is running if (containerSignatureMatcher.isExactMatch( highestPriRequest.getCookie().getContainerSignature(), @@ -1254,11 +1252,11 @@ boolean preemptIfNeeded() { + " numRequests: " + numHighestPriRequests + ". " + constructPreemptionPeriodicLog(freeResources)); } - + // highest priority request will not fit in existing free resources // free up some more // TODO this is subject to error wrt RM resource normalization - + numPendingRequestsToService = scaleDownByPreemptionPercentage(numHighestPriRequests, preemptionPercentage); @@ -1273,7 +1271,7 @@ boolean preemptIfNeeded() { + highestPriRequest.getPriority()); } - for (int i=0; i entry : taskAllocations.entrySet()) { + for (Map.Entry entry : taskAllocations.entrySet()) { HeldContainer heldContainer = heldContainers.get(entry.getValue().getId()); CookieContainerRequest lastTaskInfo = heldContainer.getLastTaskInfo(); Priority taskPriority = lastTaskInfo.getPriority(); Object signature = lastTaskInfo.getCookie().getContainerSignature(); - if(!isHigherPriority(highestPriRequest.getPriority(), taskPriority)) { + if (!isHigherPriority(highestPriRequest.getPriority(), taskPriority)) { // higher or same priority continue; } @@ -1364,7 +1362,7 @@ boolean preemptIfNeeded() { preemptedTaskPriority = taskPriority; } } - if(preemptedTaskPriority != null) { + if (preemptedTaskPriority != null) { int newNumPendingRequestsToService = scaleDownByPreemptionPercentage(Math.min( numEntriesAtPreemptedPriority, numHighestPriRequests), preemptionPercentage); numPendingRequestsToService = Math.min(newNumPendingRequestsToService, @@ -1399,10 +1397,10 @@ boolean preemptIfNeeded() { // and we get its completed container status } } - + // upcall outside locks if (preemptedContainers != null) { - for(int i=0; i> pRequestsList = - amRmClient.getMatchingRequestsForTopPriority(location, capability); - if (considerContainerAffinity && + amRmClient.getMatchingRequestsForTopPriority(location, capability); + if (considerContainerAffinity && !priorityHasAffinity.contains(amRmClient.getTopPriority())) { considerContainerAffinity = false; } @@ -1483,7 +1481,7 @@ private CookieContainerRequest getMatchingRequestWithoutPriority( // skip the expensive canAssignTaskToContainer() if the request is // not affinitized to the container container.getId().equals(cookieContainerRequest.getAffinitizedContainer()) - ) { + ) { if (canAssignTaskToContainer(cookieContainerRequest, container)) { // request matched to container if (!considerContainerAffinity) { @@ -1491,7 +1489,7 @@ private CookieContainerRequest getMatchingRequestWithoutPriority( } ContainerId affCId = cookieContainerRequest.getAffinitizedContainer(); boolean canMatchTaskWithAffinity = true; - if (affCId == null || + if (affCId == null || !heldContainers.containsKey(affCId) || inUseContainers.contains(affCId)) { // affinity not specified @@ -1503,7 +1501,7 @@ private CookieContainerRequest getMatchingRequestWithoutPriority( if (container.getId().equals( cookieContainerRequest.getAffinitizedContainer())) { // container level match - LOG.debug("Matching with affinity for request: {} container: {}", + LOG.debug("Matching with affinity for request: {} container: {}", cookieContainerRequest, affCId); return cookieContainerRequest; } @@ -1519,7 +1517,7 @@ private CookieContainerRequest getMatchingRequestWithoutPriority( } } } - + return firstMatch; } @@ -1544,14 +1542,14 @@ private boolean canAssignTaskToContainer( .getContainerSignature())) { if (LOG.isDebugEnabled()) { LOG.debug("Matched delayed container to task" - + " containerId=" + heldContainer.container.getId()); + + " containerId=" + heldContainer.container.getId()); } return true; } } if (LOG.isDebugEnabled()) { LOG.debug("Failed to match delayed container to task" - + " containerId=" + heldContainer.container.getId()); + + " containerId=" + heldContainer.container.getId()); } return false; } @@ -1582,8 +1580,8 @@ private void releaseContainer(ContainerId containerId) { } private void assignContainer(Object task, - Container container, - CookieContainerRequest assigned) { + Container container, + CookieContainerRequest assigned) { CookieContainerRequest request = removeTaskRequest(task); assert request != null; //assert assigned.equals(request); @@ -1592,10 +1590,10 @@ private void assignContainer(Object task, assert result == null; inUseContainers.add(container.getId()); containerAssignments.put(container.getId(), task); - HeldContainer heldContainer = heldContainers.get(container.getId()); + HeldContainer heldContainer = heldContainers.get(container.getId()); if (!shouldReuseContainers && heldContainer == null) { heldContainers.put(container.getId(), new HeldContainer(container, - -1, -1, assigned, this.containerSignatureMatcher)); + -1, -1, assigned, this.containerSignatureMatcher)); Resources.addTo(allocatedResources, container.getResource()); } else { if (heldContainer.isNew()) { @@ -1610,8 +1608,8 @@ private void assignContainer(Object task, heldContainer.setLastTaskInfo(assigned); } } - - private void pushNewContainerToDelayed(List containers){ + + private void pushNewContainerToDelayed(List containers) { long expireTime = getHeldContainerExpireTime(System.currentTimeMillis()); synchronized (delayedContainerManager) { @@ -1627,15 +1625,15 @@ private void pushNewContainerToDelayed(List containers){ } Resources.addTo(allocatedResources, container.getResource()); delayedContainerManager.addDelayedContainer(container, - nextScheduleTime + 1); + nextScheduleTime + 1); } } - delayedContainerManager.triggerScheduling(false); + delayedContainerManager.triggerScheduling(false); } private CookieContainerRequest removeTaskRequest(Object task) { CookieContainerRequest request = taskRequests.remove(task); - if(request != null) { + if (request != null) { // remove all references of the request from AMRMClient amRmClient.removeContainerRequest(request); } @@ -1643,7 +1641,7 @@ private CookieContainerRequest removeTaskRequest(Object task) { } private void addTaskRequest(Object task, - CookieContainerRequest request) { + CookieContainerRequest request) { CookieContainerRequest oldRequest = taskRequests.put(task, request); if (oldRequest != null) { // remove all references of the request from AMRMClient @@ -1662,17 +1660,17 @@ private Container doBookKeepingForTaskDeallocate(Object task) { } private Object unAssignContainer(ContainerId containerId, - boolean releaseIfFound) { + boolean releaseIfFound) { // Not removing. containerAssignments tracks the last task run on a // container. Object task = containerAssignments.get(containerId); - if(task == null) { + if (task == null) { return null; } Container container = taskAllocations.remove(task); assert container != null; inUseContainers.remove(containerId); - if(releaseIfFound) { + if (releaseIfFound) { releaseContainer(containerId); } return task; @@ -1691,7 +1689,7 @@ private synchronized void assignNewContainersWithLocation( while (containerIterator.hasNext()) { Container container = containerIterator.next(); CookieContainerRequest assigned = - assigner.assignNewContainer(container); + assigner.assignNewContainer(container); if (assigned != null) { assignedContainers.put(assigned, container); containerIterator.remove(); @@ -1716,10 +1714,10 @@ private synchronized void assignReUsedContainersWithLocation( } private synchronized boolean assignReUsedContainerWithLocation( - Container container, - ContainerAssigner assigner, - Map assignedContainers, - boolean honorLocality) { + Container container, + ContainerAssigner assigner, + Map assignedContainers, + boolean honorLocality) { Priority containerPriority = container.getPriority(); Priority topPendingTaskPriority = amRmClient.getTopPriority(); @@ -1727,8 +1725,8 @@ private synchronized boolean assignReUsedContainerWithLocation( // nothing left to assign return false; } - - if (topPendingTaskPriority.compareTo(containerPriority) > 0 && + + if (topPendingTaskPriority.compareTo(containerPriority) > 0 && heldContainers.get(container.getId()).isNew()) { // if the next task to assign is higher priority than the container then // dont assign this container to that task. @@ -1748,9 +1746,9 @@ private synchronized boolean assignReUsedContainerWithLocation( // re-used then this is not relevant return false; } - + CookieContainerRequest assigned = - assigner.assignReUsedContainer(container, honorLocality); + assigner.assignReUsedContainer(container, honorLocality); if (assigned != null) { assignedContainers.put(assigned, container); return true; @@ -1768,7 +1766,7 @@ private void releaseUnassignedContainers(Iterable containers) { } private void informAppAboutAssignment(CookieContainerRequest assigned, - Container container) { + Container container) { getContext().taskAllocated(getTask(assigned), assigned.getCookie().getAppCookie(), container); } @@ -1786,21 +1784,21 @@ private void informAppAboutAssignments( if (blacklistedNodes.contains(container.getNodeId())) { CookieContainerRequest request = entry.getKey(); Object task = getTask(request); - LOG.info("Container: " + container.getId() + - " allocated on blacklisted node: " + container.getNodeId() + + LOG.info("Container: " + container.getId() + + " allocated on blacklisted node: " + container.getNodeId() + " for task: " + task); Object deAllocTask = deallocateContainer(container.getId()); assert deAllocTask.equals(task); // its ok to submit the same request again because the RM will not give us // the bad/unhealthy nodes again. The nodes may become healthy/unblacklisted // and so its better to give the RM the full information. - allocateTask(task, request.getCapability(), - (request.getNodes() == null ? null : - request.getNodes().toArray(new String[request.getNodes().size()])), - (request.getRacks() == null ? null : - request.getRacks().toArray(new String[request.getRacks().size()])), - request.getPriority(), - request.getCookie().getContainerSignature(), + allocateTask(task, request.getCapability(), + (request.getNodes() == null ? null : + request.getNodes().toArray(new String[request.getNodes().size()])), + (request.getRacks() == null ? null : + request.getRacks().toArray(new String[request.getRacks().size()])), + request.getPriority(), + request.getCookie().getContainerSignature(), request.getCookie().getAppCookie()); } else { informAppAboutAssignment(entry.getKey(), container); @@ -1820,7 +1818,7 @@ public abstract CookieContainerRequest assignNewContainer( Container container); public abstract CookieContainerRequest assignReUsedContainer( - Container container, boolean honorLocality); + Container container, boolean honorLocality); public void doBookKeepingForAssignedContainer( CookieContainerRequest assigned, Container container, @@ -1832,21 +1830,21 @@ public void doBookKeepingForAssignedContainer( assert task != null; LOG.info("Assigning container to task: " - + "containerId=" + container.getId() - + ", task=" + task - + ", containerHost=" + container.getNodeId() - + ", containerPriority= " + container.getPriority() - + ", containerResources=" + container.getResource() - + ", localityMatchType=" + locality - + ", matchedLocation=" + matchedLocation - + ", honorLocalityFlags=" + honorLocalityFlags - + ", reusedContainer=" + containerAssignments.containsKey(container.getId()) - + ", delayedContainers=" + delayedContainerManager.delayedContainers.size()); + + "containerId=" + container.getId() + + ", task=" + task + + ", containerHost=" + container.getNodeId() + + ", containerPriority= " + container.getPriority() + + ", containerResources=" + container.getResource() + + ", localityMatchType=" + locality + + ", matchedLocation=" + matchedLocation + + ", honorLocalityFlags=" + honorLocalityFlags + + ", reusedContainer=" + containerAssignments.containsKey(container.getId()) + + ", delayedContainers=" + delayedContainerManager.delayedContainers.size()); assignContainer(task, container, assigned); } } - + private class NodeLocalContainerAssigner extends ContainerAssigner { NodeLocalContainerAssigner() { @@ -1864,13 +1862,12 @@ public CookieContainerRequest assignNewContainer(Container container) { @Override public CookieContainerRequest assignReUsedContainer(Container container, - boolean honorLocality) { + boolean honorLocality) { String location = container.getNodeId().getHost(); CookieContainerRequest assigned = getMatchingRequestWithoutPriority( - container, location, true); + container, location, true); doBookKeepingForAssignedContainer(assigned, container, location, true); return assigned; - } } @@ -1892,7 +1889,7 @@ public CookieContainerRequest assignNewContainer(Container container) { @Override public CookieContainerRequest assignReUsedContainer( - Container container, boolean honorLocality) { + Container container, boolean honorLocality) { // TEZ-586 this is not match an actual rackLocal request unless honorLocality // is false. This method is useless if honorLocality=true if (!honorLocality) { @@ -1924,21 +1921,19 @@ public CookieContainerRequest assignNewContainer(Container container) { @Override public CookieContainerRequest assignReUsedContainer(Container container, - boolean honorLocality) { + boolean honorLocality) { if (!honorLocality) { String location = ResourceRequest.ANY; CookieContainerRequest assigned = getMatchingRequestWithoutPriority( - container, location, false); + container, location, false); doBookKeepingForAssignedContainer(assigned, container, location, honorLocality); return assigned; } return null; } - } - - + @VisibleForTesting class DelayedContainerManager extends Thread { @@ -1946,19 +1941,19 @@ class HeldContainerTimerComparator implements Comparator { @Override public int compare(HeldContainer c1, - HeldContainer c2) { + HeldContainer c2) { return (int) (c1.getNextScheduleTime() - c2.getNextScheduleTime()); } } PriorityBlockingQueue delayedContainers = - new PriorityBlockingQueue(20, - new HeldContainerTimerComparator()); + new PriorityBlockingQueue(20, + new HeldContainerTimerComparator()); private volatile boolean tryAssigningAll = false; private volatile boolean running = true; private long maxScheduleTimeSeen = -1; - + // used for testing only @VisibleForTesting volatile AtomicBoolean drainedDelayedContainersForTest = null; @@ -1966,7 +1961,7 @@ public int compare(HeldContainer c1, DelayedContainerManager() { super.setName("DelayedContainerManager"); } - + @Override public void run() { try { @@ -1977,7 +1972,7 @@ public void run() { } private void mainLoop() { - while(running) { + while (running) { // Try assigning all containers if there's a request to do so. if (tryAssigningAll) { doAssignAll(); @@ -1987,7 +1982,7 @@ private void mainLoop() { // Try allocating containers which have timed out. // Required since these containers may get assigned without // locality at this point. - synchronized(this) { + synchronized (this) { if (delayedContainers.peek() == null) { try { // test only signaling to make TestTaskScheduler work @@ -2022,7 +2017,7 @@ private void mainLoop() { long nextScheduleTs = delayedContainer.getNextScheduleTime(); if (currentTs >= nextScheduleTs) { Map assignedContainers = null; - synchronized(YarnTaskSchedulerService.this) { + synchronized (YarnTaskSchedulerService.this) { // Remove the container and try scheduling it. // TEZ-587 what if container is released by RM after this // in onContainerCompleted() @@ -2043,7 +2038,7 @@ private void mainLoop() { // Inform App should be done outside of the lock informAppAboutAssignments(assignedContainers); } else { - synchronized(this) { + synchronized (this) { try { // Wait for the next container to be assignable delayedContainer = delayedContainers.peek(); @@ -2062,7 +2057,7 @@ private void mainLoop() { } releasePendingContainers(); } - + private void doAssignAll() { // The allocatedContainers queue should not be modified in the middle of an iteration over it. // Synchronizing here on TaskScheduler.this to prevent this from happening. @@ -2076,36 +2071,36 @@ private void doAssignAll() { } Map assignedContainers; - synchronized(YarnTaskSchedulerService.this) { + synchronized (YarnTaskSchedulerService.this) { // honor reuse-locality flags (container not timed out yet), Don't queue // (already in queue), don't release (release happens when containers // time-out) LOG.debug("Trying to assign all delayed containers to newly received tasks"); Iterator iter = delayedContainers.iterator(); - while(iter.hasNext()) { + while (iter.hasNext()) { HeldContainer delayedContainer = iter.next(); if (!heldContainers.containsKey(delayedContainer.getContainer().getId())) { // this container is no longer held by us // non standard scenario LOG.info("AssignAll - Skipping delayed container as container is no longer" - + " running, containerId=" - + delayedContainer.getContainer().getId()); + + " running, containerId=" + + delayedContainer.getContainer().getId()); iter.remove(); } } assignedContainers = tryAssignReUsedContainers( - new ContainerIterable(delayedContainers)); + new ContainerIterable(delayedContainers)); } // Inform app informAppAboutAssignments(assignedContainers); } - + /** * Indicate that an attempt should be made to allocate all available containers. * Intended to be used in cases where new Container requests come in */ public void triggerScheduling(boolean scheduleAll) { - synchronized(this) { + synchronized (this) { this.tryAssigningAll = scheduleAll; this.notify(); } @@ -2115,17 +2110,17 @@ public void shutdown() { this.running = false; this.interrupt(); } - + private void releasePendingContainers() { List pendingContainers = Lists.newArrayListWithCapacity( - delayedContainers.size()); + delayedContainers.size()); delayedContainers.drainTo(pendingContainers); releaseUnassignedContainers(new ContainerIterable(pendingContainers)); } @VisibleForTesting void addDelayedContainer(Container container, - long nextScheduleTime) { + long nextScheduleTime) { HeldContainer delayedContainer = heldContainers.get(container.getId()); if (delayedContainer == null) { LOG.warn("Attempting to add a non-running container to the" @@ -2139,12 +2134,12 @@ void addDelayedContainer(Container container, } if (LOG.isDebugEnabled()) { LOG.debug("Adding container to delayed queue" - + ", containerId=" + delayedContainer.getContainer().getId() - + ", nextScheduleTime=" + delayedContainer.getNextScheduleTime() - + ", containerExpiry=" + delayedContainer.getContainerExpiryTime()); + + ", containerId=" + delayedContainer.getContainer().getId() + + ", nextScheduleTime=" + delayedContainer.getNextScheduleTime() + + ", containerExpiry=" + delayedContainer.getContainerExpiryTime()); } - boolean added = false; - synchronized(this) { + boolean added = false; + synchronized (this) { added = delayedContainers.offer(delayedContainer); if (drainedDelayedContainersForTest != null) { synchronized (drainedDelayedContainersForTest) { @@ -2157,22 +2152,21 @@ void addDelayedContainer(Container container, releaseUnassignedContainers(Lists.newArrayList(container)); } } - } - + synchronized void determineMinHeldContainers() { sessionMinHeldContainers.clear(); if (sessionNumMinHeldContainers <= 0) { return; } - + if (heldContainers.size() <= sessionNumMinHeldContainers) { sessionMinHeldContainers.addAll(heldContainers.keySet()); } - + Map rackHeldNumber = Maps.newHashMap(); Map> nodeHeldContainers = Maps.newHashMap(); - for(HeldContainer heldContainer : heldContainers.values()) { + for (HeldContainer heldContainer : heldContainers.values()) { AtomicInteger count = rackHeldNumber.get(heldContainer.getRack()); if (count == null) { count = new AtomicInteger(0); @@ -2190,7 +2184,7 @@ synchronized void determineMinHeldContainers() { for (String rack : rackHeldNumber.keySet()) { rackToHoldNumber.put(rack, new AtomicInteger(0)); } - + // distribute evenly across nodes // the loop assigns 1 container per rack over all racks int containerCount = 0; @@ -2198,7 +2192,7 @@ synchronized void determineMinHeldContainers() { Iterator> iter = rackHeldNumber.entrySet().iterator(); while (containerCount < sessionNumMinHeldContainers && iter.hasNext()) { Entry entry = iter.next(); - if (entry.getValue().decrementAndGet() >=0) { + if (entry.getValue().decrementAndGet() >= 0) { containerCount++; rackToHoldNumber.get(entry.getKey()).incrementAndGet(); } else { @@ -2206,7 +2200,7 @@ synchronized void determineMinHeldContainers() { } } } - + // distribute containers evenly across nodes while not exceeding rack limit // the loop assigns 1 container per node over all nodes containerCount = 0; @@ -2230,7 +2224,7 @@ synchronized void determineMinHeldContainers() { } } } - + LOG.info("Holding on to " + sessionMinHeldContainers.size() + " containers" + " out of total held containers: " + heldContainers.size()); } @@ -2287,12 +2281,12 @@ enum LocalityMatchLevel { private int numAssignmentAttempts = 0; private Object lastAssignedContainerSignature; final ContainerSignatureMatcher signatureMatcher; - + HeldContainer(Container container, - long nextScheduleTime, - long containerExpiryTime, - CookieContainerRequest firstTaskInfo, - ContainerSignatureMatcher signatureMatcher) { + long nextScheduleTime, + long containerExpiryTime, + CookieContainerRequest firstTaskInfo, + ContainerSignatureMatcher signatureMatcher) { this.container = container; this.nextScheduleTime = nextScheduleTime; if (firstTaskInfo != null) { @@ -2305,35 +2299,35 @@ enum LocalityMatchLevel { .getNetworkLocation(); this.signatureMatcher = signatureMatcher; } - + boolean isNew() { return lastTaskInfo == null; } - + String getRack() { return this.rack; } - + String getNode() { return this.container.getNodeId().getHost(); } - + int geNumAssignmentAttempts() { return numAssignmentAttempts; } - + void incrementAssignmentAttempts() { numAssignmentAttempts++; } - + public Container getContainer() { return this.container; } - + public long getNextScheduleTime() { return this.nextScheduleTime; } - + public void setNextScheduleTime(long nextScheduleTime) { this.nextScheduleTime = nextScheduleTime; } @@ -2353,7 +2347,7 @@ public Object getLastAssignedContainerSignature() { public CookieContainerRequest getLastTaskInfo() { return this.lastTaskInfo; } - + public void setLastTaskInfo(CookieContainerRequest taskInfo) { // Merge the container signatures to account for any changes to the container // footprint. For example, re-localization of additional resources will @@ -2381,7 +2375,7 @@ public synchronized void incrementLocalityMatchLevel() { localityMatchLevel = LocalityMatchLevel.NON_LOCAL; } else if (localityMatchLevel.equals(LocalityMatchLevel.NON_LOCAL)) { throw new TezUncheckedException("Cannot increment locality level " - + " from current NON_LOCAL for container: " + container.getId()); + + " from current NON_LOCAL for container: " + container.getId()); } } @@ -2395,8 +2389,8 @@ public String toString() { + ", nextScheduleTime: " + nextScheduleTime + ", localityMatchLevel=" + localityMatchLevel + ", signature: " - + (lastAssignedContainerSignature != null? lastAssignedContainerSignature.toString() - : "null"); + + (lastAssignedContainerSignature != null ? lastAssignedContainerSignature.toString() + : "null"); } } } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerServiceError.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerServiceError.java index e8017dd305..2593e7ea26 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerServiceError.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/YarnTaskSchedulerServiceError.java @@ -29,5 +29,4 @@ public Enum getEnum() { public ErrorType getErrorType() { return ErrorType.PERMANENT; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java index 5f90a89520..86fc331988 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainer.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.rm.container; @@ -25,16 +25,25 @@ import org.apache.hadoop.yarn.event.EventHandler; import org.apache.tez.dag.records.TezTaskAttemptID; -public interface AMContainer extends EventHandler{ - +public interface AMContainer extends EventHandler { + public AMContainerState getState(); + public ContainerId getContainerId(); + public Container getContainer(); + public List getAllTaskAttempts(); + public TezTaskAttemptID getCurrentTaskAttempt(); + public long getCurrentTaskAttemptAllocationTime(); + public int getTaskSchedulerIdentifier(); + public int getContainerLauncherIdentifier(); + public int getTaskCommunicatorIdentifier(); + public boolean isInErrorState(); } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEvent.java index b47ffb0067..d2783a9fdf 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEvent.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -23,12 +23,12 @@ public class AMContainerEvent extends AbstractEvent { private final ContainerId containerId; - + public AMContainerEvent(ContainerId containerId, AMContainerEventType type) { super(type); this.containerId = containerId; } - + public ContainerId getContainerId() { return this.containerId; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java index 682cd02d07..e45cc007a4 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventAssignTA.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -35,8 +35,8 @@ public class AMContainerEventAssignTA extends AMContainerEvent { private final int priority; public AMContainerEventAssignTA(ContainerId containerId, TezTaskAttemptID attemptId, - Object remoteTaskSpec, Map taskLocalResources, Credentials credentials, - int priority) { + Object remoteTaskSpec, Map taskLocalResources, Credentials credentials, + int priority) { super(containerId, AMContainerEventType.C_ASSIGN_TA); this.attemptId = attemptId; this.remoteTaskSpec = (TaskSpec) remoteTaskSpec; @@ -48,7 +48,7 @@ public AMContainerEventAssignTA(ContainerId containerId, TezTaskAttemptID attemp public TaskSpec getRemoteTaskSpec() { return this.remoteTaskSpec; } - + public Map getRemoteTaskLocalResources() { return this.taskLocalResources; } @@ -56,7 +56,7 @@ public Map getRemoteTaskLocalResources() { public TezTaskAttemptID getTaskAttemptId() { return this.attemptId; } - + public Credentials getCredentials() { return this.credentials; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventCompleted.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventCompleted.java index cecb01974b..7031bcdf3e 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventCompleted.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventCompleted.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.rm.container; @@ -29,8 +29,8 @@ public class AMContainerEventCompleted extends AMContainerEvent { private final String diagnostics; private final TaskAttemptTerminationCause errCause; - public AMContainerEventCompleted(ContainerId containerId, - int exitStatus, String diagnostics, TaskAttemptTerminationCause errCause) { + public AMContainerEventCompleted(ContainerId containerId, + int exitStatus, String diagnostics, TaskAttemptTerminationCause errCause) { super(containerId, AMContainerEventType.C_COMPLETED); this.exitStatus = exitStatus; this.diagnostics = diagnostics; @@ -38,26 +38,26 @@ public AMContainerEventCompleted(ContainerId containerId, } public boolean isPreempted() { - return (exitStatus == ContainerExitStatus.PREEMPTED || + return (exitStatus == ContainerExitStatus.PREEMPTED || errCause == TaskAttemptTerminationCause.INTERNAL_PREEMPTION); } - + public boolean isDiskFailed() { return (exitStatus == ContainerExitStatus.DISKS_FAILED); } - + public boolean isSystemAction() { return isPreempted() || isDiskFailed(); } - + public String getDiagnostics() { return diagnostics; } - + public int getContainerExitStatus() { return exitStatus; } - + public TaskAttemptTerminationCause getTerminationCause() { return errCause; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchFailed.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchFailed.java index 7d476d9443..160a1fb30c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchFailed.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchFailed.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -22,15 +22,14 @@ public class AMContainerEventLaunchFailed extends AMContainerEvent { private final String message; - + public AMContainerEventLaunchFailed(ContainerId containerId, - String message) { + String message) { super(containerId, AMContainerEventType.C_LAUNCH_FAILED); this.message = message; } - + public String getMessage() { return this.message; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchRequest.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchRequest.java index c3aae35910..2c0ee1939e 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchRequest.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunchRequest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,8 +31,8 @@ public class AMContainerEventLaunchRequest extends AMContainerEvent { private final int taskCommId; public AMContainerEventLaunchRequest(ContainerId containerId, - TezVertexID vertexId, ContainerContext containerContext, - int launcherId, int taskCommId) { + TezVertexID vertexId, ContainerContext containerContext, + int launcherId, int taskCommId) { super(containerId, AMContainerEventType.C_LAUNCH_REQUEST); this.vertexId = vertexId; this.containerContext = containerContext; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunched.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunched.java index 7a8ecd3035..e2b9429bad 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunched.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventLaunched.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -24,5 +24,4 @@ public class AMContainerEventLaunched extends AMContainerEvent { public AMContainerEventLaunched(ContainerId containerId) { super(containerId, AMContainerEventType.C_LAUNCHED); } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventNodeFailed.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventNodeFailed.java index 9f5d39ce10..aa95692d12 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventNodeFailed.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventNodeFailed.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -35,5 +35,4 @@ public AMContainerEventNodeFailed(ContainerId containerId, String message) { public String getDiagnosticInfo() { return message; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventStopFailed.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventStopFailed.java index 3642edb184..28055064a1 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventStopFailed.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventStopFailed.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -23,7 +23,7 @@ public class AMContainerEventStopFailed extends AMContainerEvent { // TODO XXX Not being used for anything. May be useful if we rely less on // the RM informing the job about container failure. - + private final String message; public AMContainerEventStopFailed(ContainerId containerId, String message) { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventStopRequest.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventStopRequest.java index e2930af684..f477b38b21 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventStopRequest.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventStopRequest.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -24,5 +24,4 @@ public class AMContainerEventStopRequest extends AMContainerEvent { public AMContainerEventStopRequest(ContainerId containerId) { super(containerId, AMContainerEventType.C_STOP_REQUEST); } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventTASucceeded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventTASucceeded.java index e0c8fe72fc..32c5ce5248 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventTASucceeded.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventTASucceeded.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -25,11 +25,11 @@ public class AMContainerEventTASucceeded extends AMContainerEvent { private final TezTaskAttemptID attemptId; public AMContainerEventTASucceeded(ContainerId containerId, - TezTaskAttemptID attemptId) { + TezTaskAttemptID attemptId) { super(containerId, AMContainerEventType.C_TA_SUCCEEDED); this.attemptId = attemptId; } - + public TezTaskAttemptID getTaskAttemptId() { return this.attemptId; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventType.java index 330ad57c4e..ab781b2fa8 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventType.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerEventType.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.rm.container; @@ -23,7 +23,7 @@ public enum AMContainerEventType { //Producer: Scheduler C_LAUNCH_REQUEST, C_ASSIGN_TA, - + //Producer: NMCommunicator C_LAUNCHED, C_LAUNCH_FAILED, @@ -33,21 +33,21 @@ public enum AMContainerEventType { //Producer: RMCommunicator C_COMPLETED, - + //Producer: RMCommunicator, AMNode C_NODE_FAILED, - + //TODO ZZZ CREUSE: Consider introducing a new event C_NODE_BLACKLISTED -> container can take a call on what to do if this event comes in. - + //Producer: TA-> Scheduler -> Container (in case of failure etc) // Scheduler -> Container (in case of pre-emption etc) // Node -> Container (in case of Node blacklisted etc) C_STOP_REQUEST, - + //Producer: NMCommunicator C_NM_STOP_FAILED, C_NM_STOP_SENT, - + //Producer: ContainerHeartbeatHandler C_TIMED_OUT, } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java index 19cf5b7e21..c3f4333820 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.rm.container; @@ -75,7 +75,7 @@ public static void dagComplete(TezDAGID dagId) { * Create a {@link LocalResource} record with all the given parameters. */ public static LocalResource createLocalResource(FileSystem fc, Path file, - LocalResourceType type, LocalResourceVisibility visibility) + LocalResourceType type, LocalResourceVisibility visibility) throws IOException { FileStatus fstat = fc.getFileStatus(file); URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat @@ -104,12 +104,12 @@ private static ContainerLaunchContext createCommonContainerLaunchContext( Map serviceData = new HashMap(); // Tokens - + // Setup up task credentials buffer - ByteBuffer containerCredentialsBuffer = ByteBuffer.wrap(new byte[] {}); + ByteBuffer containerCredentialsBuffer = ByteBuffer.wrap(new byte[]{}); try { Credentials containerCredentials = new Credentials(); - + // All Credentials need to be set so that YARN can localize the resources // correctly, even though they may not be used by all tasks which will run // on this container. @@ -185,9 +185,9 @@ public static ContainerLaunchContext createContainerLaunchContext( if (LOG.isDebugEnabled()) { if (!modifiedJavaOpts.equals(javaOpts)) { LOG.debug("Modified java opts for container" - + ", containerId=" + containerId - + ", originalJavaOpts=" + javaOpts - + ", modifiedJavaOpts=" + modifiedJavaOpts); + + ", containerId=" + containerId + + ", originalJavaOpts=" + javaOpts + + ", modifiedJavaOpts=" + modifiedJavaOpts); } } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java index 6b67eb9da1..87c0810743 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.rm.container; @@ -102,7 +102,7 @@ public class AMContainerImpl implements AMContainer { private long lastTaskFinishTime; private TezDAGID lastTaskDAGID; - + // An assign can happen even during wind down. e.g. NodeFailure caused the // wind down, and an allocation was pending in the AMScheduler. This could // be modelled as a separate state. @@ -123,7 +123,7 @@ public class AMContainerImpl implements AMContainer { private boolean credentialsChanged = false; private boolean completedMessageSent = false; - + // TODO Consider registering with the TAL, instead of the TAL pulling. // Possibly after splitting TAL and ContainerListener. @@ -141,178 +141,176 @@ public class AMContainerImpl implements AMContainer { stateMachineFactory = new StateMachineFactory( - AMContainerState.ALLOCATED) - - .addTransition(AMContainerState.ALLOCATED, AMContainerState.LAUNCHING, - AMContainerEventType.C_LAUNCH_REQUEST, new LaunchRequestTransition()) - .addTransition(AMContainerState.ALLOCATED, AMContainerState.COMPLETED, - AMContainerEventType.C_ASSIGN_TA, - new AssignTaskAttemptAtAllocatedTransition()) - .addTransition(AMContainerState.ALLOCATED, AMContainerState.COMPLETED, - AMContainerEventType.C_COMPLETED, - new CompletedAtAllocatedTransition()) - .addTransition(AMContainerState.ALLOCATED, AMContainerState.COMPLETED, - AMContainerEventType.C_STOP_REQUEST, - new StopRequestAtAllocatedTransition()) - .addTransition(AMContainerState.ALLOCATED, AMContainerState.COMPLETED, - AMContainerEventType.C_NODE_FAILED, - new NodeFailedAtAllocatedTransition()) - .addTransition( - AMContainerState.ALLOCATED, - AMContainerState.COMPLETED, - EnumSet.of(AMContainerEventType.C_LAUNCHED, - AMContainerEventType.C_LAUNCH_FAILED, - AMContainerEventType.C_TA_SUCCEEDED, - AMContainerEventType.C_NM_STOP_SENT, - AMContainerEventType.C_NM_STOP_FAILED, - AMContainerEventType.C_TIMED_OUT), new ErrorTransition()) - .addTransition( - AMContainerState.LAUNCHING, - EnumSet.of(AMContainerState.LAUNCHING, AMContainerState.STOP_REQUESTED), - AMContainerEventType.C_ASSIGN_TA, new AssignTaskAttemptTransition()) - .addTransition(AMContainerState.LAUNCHING, - EnumSet.of(AMContainerState.IDLE, AMContainerState.RUNNING), - AMContainerEventType.C_LAUNCHED, new LaunchedTransition()) - .addTransition(AMContainerState.LAUNCHING, AMContainerState.STOPPING, - AMContainerEventType.C_LAUNCH_FAILED, new LaunchFailedTransition()) - // Is assuming the pullAttempt will be null. - .addTransition(AMContainerState.LAUNCHING, AMContainerState.COMPLETED, - AMContainerEventType.C_COMPLETED, - new CompletedAtLaunchingTransition()) - .addTransition(AMContainerState.LAUNCHING, - AMContainerState.STOP_REQUESTED, AMContainerEventType.C_STOP_REQUEST, - new StopRequestAtLaunchingTransition()) - .addTransition(AMContainerState.LAUNCHING, AMContainerState.STOPPING, - AMContainerEventType.C_NODE_FAILED, - new NodeFailedAtLaunchingTransition()) - .addTransition( - AMContainerState.LAUNCHING, - AMContainerState.STOP_REQUESTED, - EnumSet.of(AMContainerEventType.C_LAUNCH_REQUEST, - AMContainerEventType.C_TA_SUCCEEDED, - AMContainerEventType.C_NM_STOP_SENT, - AMContainerEventType.C_NM_STOP_FAILED, - AMContainerEventType.C_TIMED_OUT), - new ErrorAtLaunchingTransition()) - - .addTransition(AMContainerState.IDLE, - EnumSet.of(AMContainerState.RUNNING, AMContainerState.STOP_REQUESTED), - AMContainerEventType.C_ASSIGN_TA, - new AssignTaskAttemptTransition()) - .addTransition(AMContainerState.IDLE, AMContainerState.COMPLETED, - AMContainerEventType.C_COMPLETED, new CompletedAtIdleTransition()) - .addTransition(AMContainerState.IDLE, AMContainerState.STOP_REQUESTED, - AMContainerEventType.C_STOP_REQUEST, - new StopRequestAtIdleTransition()) - .addTransition(AMContainerState.IDLE, AMContainerState.STOP_REQUESTED, - AMContainerEventType.C_TIMED_OUT, new TimedOutAtIdleTransition()) - .addTransition(AMContainerState.IDLE, AMContainerState.STOPPING, - AMContainerEventType.C_NODE_FAILED, new NodeFailedAtIdleTransition()) - .addTransition( - AMContainerState.IDLE, - AMContainerState.STOP_REQUESTED, - EnumSet.of(AMContainerEventType.C_LAUNCH_REQUEST, - AMContainerEventType.C_LAUNCHED, - AMContainerEventType.C_LAUNCH_FAILED, - AMContainerEventType.C_TA_SUCCEEDED, - AMContainerEventType.C_NM_STOP_SENT, - AMContainerEventType.C_NM_STOP_FAILED), - new ErrorAtIdleTransition()) - - .addTransition(AMContainerState.RUNNING, AMContainerState.STOP_REQUESTED, - AMContainerEventType.C_ASSIGN_TA, - new AssignTaskAttemptAtRunningTransition()) - .addTransition(AMContainerState.RUNNING, AMContainerState.IDLE, - AMContainerEventType.C_TA_SUCCEEDED, - new TASucceededAtRunningTransition()) - .addTransition(AMContainerState.RUNNING, AMContainerState.COMPLETED, - AMContainerEventType.C_COMPLETED, new CompletedAtRunningTransition()) - .addTransition(AMContainerState.RUNNING, AMContainerState.STOP_REQUESTED, - AMContainerEventType.C_STOP_REQUEST, - new StopRequestAtRunningTransition()) - .addTransition(AMContainerState.RUNNING, AMContainerState.STOP_REQUESTED, - AMContainerEventType.C_TIMED_OUT, new TimedOutAtRunningTransition()) - .addTransition(AMContainerState.RUNNING, AMContainerState.STOPPING, - AMContainerEventType.C_NODE_FAILED, - new NodeFailedAtRunningTransition()) - .addTransition( - AMContainerState.RUNNING, - AMContainerState.STOP_REQUESTED, - EnumSet.of(AMContainerEventType.C_LAUNCH_REQUEST, - AMContainerEventType.C_LAUNCHED, - AMContainerEventType.C_LAUNCH_FAILED, - AMContainerEventType.C_NM_STOP_SENT, - AMContainerEventType.C_NM_STOP_FAILED), - new ErrorAtRunningTransition()) - - .addTransition(AMContainerState.STOP_REQUESTED, - AMContainerState.STOP_REQUESTED, AMContainerEventType.C_ASSIGN_TA, - new AssignTAAtWindDownTransition()) - .addTransition(AMContainerState.STOP_REQUESTED, - AMContainerState.COMPLETED, AMContainerEventType.C_COMPLETED, - new CompletedAtWindDownTransition()) - .addTransition(AMContainerState.STOP_REQUESTED, - AMContainerState.STOPPING, AMContainerEventType.C_NM_STOP_SENT) - .addTransition(AMContainerState.STOP_REQUESTED, - AMContainerState.STOPPING, AMContainerEventType.C_NM_STOP_FAILED, - new NMStopRequestFailedTransition()) - .addTransition(AMContainerState.STOP_REQUESTED, - AMContainerState.STOPPING, AMContainerEventType.C_NODE_FAILED, - new NodeFailedAtNMStopRequestedTransition()) - .addTransition( - AMContainerState.STOP_REQUESTED, - AMContainerState.STOP_REQUESTED, - EnumSet.of(AMContainerEventType.C_LAUNCHED, - AMContainerEventType.C_LAUNCH_FAILED, - AMContainerEventType.C_TA_SUCCEEDED, + AMContainerState.ALLOCATED) + + .addTransition(AMContainerState.ALLOCATED, AMContainerState.LAUNCHING, + AMContainerEventType.C_LAUNCH_REQUEST, new LaunchRequestTransition()) + .addTransition(AMContainerState.ALLOCATED, AMContainerState.COMPLETED, + AMContainerEventType.C_ASSIGN_TA, + new AssignTaskAttemptAtAllocatedTransition()) + .addTransition(AMContainerState.ALLOCATED, AMContainerState.COMPLETED, + AMContainerEventType.C_COMPLETED, + new CompletedAtAllocatedTransition()) + .addTransition(AMContainerState.ALLOCATED, AMContainerState.COMPLETED, AMContainerEventType.C_STOP_REQUEST, - AMContainerEventType.C_TIMED_OUT)) - .addTransition(AMContainerState.STOP_REQUESTED, - AMContainerState.STOP_REQUESTED, - AMContainerEventType.C_LAUNCH_REQUEST, - new ErrorAtNMStopRequestedTransition()) - - - - .addTransition(AMContainerState.STOPPING, AMContainerState.STOPPING, - AMContainerEventType.C_ASSIGN_TA, new AssignTAAtWindDownTransition()) - // TODO This transition is wrong. Should be a noop / error. - .addTransition(AMContainerState.STOPPING, AMContainerState.COMPLETED, - AMContainerEventType.C_COMPLETED, new CompletedAtWindDownTransition()) - .addTransition(AMContainerState.STOPPING, AMContainerState.STOPPING, - AMContainerEventType.C_NODE_FAILED, new NodeFailedBaseTransition()) - .addTransition( - AMContainerState.STOPPING, - AMContainerState.STOPPING, - EnumSet.of(AMContainerEventType.C_LAUNCHED, - AMContainerEventType.C_LAUNCH_FAILED, - AMContainerEventType.C_TA_SUCCEEDED, + new StopRequestAtAllocatedTransition()) + .addTransition(AMContainerState.ALLOCATED, AMContainerState.COMPLETED, + AMContainerEventType.C_NODE_FAILED, + new NodeFailedAtAllocatedTransition()) + .addTransition( + AMContainerState.ALLOCATED, + AMContainerState.COMPLETED, + EnumSet.of(AMContainerEventType.C_LAUNCHED, + AMContainerEventType.C_LAUNCH_FAILED, + AMContainerEventType.C_TA_SUCCEEDED, + AMContainerEventType.C_NM_STOP_SENT, + AMContainerEventType.C_NM_STOP_FAILED, + AMContainerEventType.C_TIMED_OUT), new ErrorTransition()) + .addTransition( + AMContainerState.LAUNCHING, + EnumSet.of(AMContainerState.LAUNCHING, AMContainerState.STOP_REQUESTED), + AMContainerEventType.C_ASSIGN_TA, new AssignTaskAttemptTransition()) + .addTransition(AMContainerState.LAUNCHING, + EnumSet.of(AMContainerState.IDLE, AMContainerState.RUNNING), + AMContainerEventType.C_LAUNCHED, new LaunchedTransition()) + .addTransition(AMContainerState.LAUNCHING, AMContainerState.STOPPING, + AMContainerEventType.C_LAUNCH_FAILED, new LaunchFailedTransition()) + // Is assuming the pullAttempt will be null. + .addTransition(AMContainerState.LAUNCHING, AMContainerState.COMPLETED, + AMContainerEventType.C_COMPLETED, + new CompletedAtLaunchingTransition()) + .addTransition(AMContainerState.LAUNCHING, + AMContainerState.STOP_REQUESTED, AMContainerEventType.C_STOP_REQUEST, + new StopRequestAtLaunchingTransition()) + .addTransition(AMContainerState.LAUNCHING, AMContainerState.STOPPING, + AMContainerEventType.C_NODE_FAILED, + new NodeFailedAtLaunchingTransition()) + .addTransition( + AMContainerState.LAUNCHING, + AMContainerState.STOP_REQUESTED, + EnumSet.of(AMContainerEventType.C_LAUNCH_REQUEST, + AMContainerEventType.C_TA_SUCCEEDED, + AMContainerEventType.C_NM_STOP_SENT, + AMContainerEventType.C_NM_STOP_FAILED, + AMContainerEventType.C_TIMED_OUT), + new ErrorAtLaunchingTransition()) + + .addTransition(AMContainerState.IDLE, + EnumSet.of(AMContainerState.RUNNING, AMContainerState.STOP_REQUESTED), + AMContainerEventType.C_ASSIGN_TA, + new AssignTaskAttemptTransition()) + .addTransition(AMContainerState.IDLE, AMContainerState.COMPLETED, + AMContainerEventType.C_COMPLETED, new CompletedAtIdleTransition()) + .addTransition(AMContainerState.IDLE, AMContainerState.STOP_REQUESTED, AMContainerEventType.C_STOP_REQUEST, - AMContainerEventType.C_NM_STOP_SENT, - AMContainerEventType.C_NM_STOP_FAILED, - AMContainerEventType.C_TIMED_OUT)) - .addTransition(AMContainerState.STOPPING, AMContainerState.STOPPING, - AMContainerEventType.C_LAUNCH_REQUEST, - new ErrorAtStoppingTransition()) - - .addTransition(AMContainerState.COMPLETED, AMContainerState.COMPLETED, - AMContainerEventType.C_ASSIGN_TA, new AssignTAAtCompletedTransition()) - .addTransition(AMContainerState.COMPLETED, AMContainerState.COMPLETED, - AMContainerEventType.C_NODE_FAILED, new NodeFailedBaseTransition()) - .addTransition( - AMContainerState.COMPLETED, - AMContainerState.COMPLETED, - EnumSet.of(AMContainerEventType.C_LAUNCH_REQUEST, - AMContainerEventType.C_LAUNCHED, - AMContainerEventType.C_LAUNCH_FAILED, + new StopRequestAtIdleTransition()) + .addTransition(AMContainerState.IDLE, AMContainerState.STOP_REQUESTED, + AMContainerEventType.C_TIMED_OUT, new TimedOutAtIdleTransition()) + .addTransition(AMContainerState.IDLE, AMContainerState.STOPPING, + AMContainerEventType.C_NODE_FAILED, new NodeFailedAtIdleTransition()) + .addTransition( + AMContainerState.IDLE, + AMContainerState.STOP_REQUESTED, + EnumSet.of(AMContainerEventType.C_LAUNCH_REQUEST, + AMContainerEventType.C_LAUNCHED, + AMContainerEventType.C_LAUNCH_FAILED, + AMContainerEventType.C_TA_SUCCEEDED, + AMContainerEventType.C_NM_STOP_SENT, + AMContainerEventType.C_NM_STOP_FAILED), + new ErrorAtIdleTransition()) + + .addTransition(AMContainerState.RUNNING, AMContainerState.STOP_REQUESTED, + AMContainerEventType.C_ASSIGN_TA, + new AssignTaskAttemptAtRunningTransition()) + .addTransition(AMContainerState.RUNNING, AMContainerState.IDLE, AMContainerEventType.C_TA_SUCCEEDED, - AMContainerEventType.C_COMPLETED, + new TASucceededAtRunningTransition()) + .addTransition(AMContainerState.RUNNING, AMContainerState.COMPLETED, + AMContainerEventType.C_COMPLETED, new CompletedAtRunningTransition()) + .addTransition(AMContainerState.RUNNING, AMContainerState.STOP_REQUESTED, AMContainerEventType.C_STOP_REQUEST, - AMContainerEventType.C_NM_STOP_SENT, - AMContainerEventType.C_NM_STOP_FAILED, - AMContainerEventType.C_TIMED_OUT)) - - .installTopology(); + new StopRequestAtRunningTransition()) + .addTransition(AMContainerState.RUNNING, AMContainerState.STOP_REQUESTED, + AMContainerEventType.C_TIMED_OUT, new TimedOutAtRunningTransition()) + .addTransition(AMContainerState.RUNNING, AMContainerState.STOPPING, + AMContainerEventType.C_NODE_FAILED, + new NodeFailedAtRunningTransition()) + .addTransition( + AMContainerState.RUNNING, + AMContainerState.STOP_REQUESTED, + EnumSet.of(AMContainerEventType.C_LAUNCH_REQUEST, + AMContainerEventType.C_LAUNCHED, + AMContainerEventType.C_LAUNCH_FAILED, + AMContainerEventType.C_NM_STOP_SENT, + AMContainerEventType.C_NM_STOP_FAILED), + new ErrorAtRunningTransition()) + + .addTransition(AMContainerState.STOP_REQUESTED, + AMContainerState.STOP_REQUESTED, AMContainerEventType.C_ASSIGN_TA, + new AssignTAAtWindDownTransition()) + .addTransition(AMContainerState.STOP_REQUESTED, + AMContainerState.COMPLETED, AMContainerEventType.C_COMPLETED, + new CompletedAtWindDownTransition()) + .addTransition(AMContainerState.STOP_REQUESTED, + AMContainerState.STOPPING, AMContainerEventType.C_NM_STOP_SENT) + .addTransition(AMContainerState.STOP_REQUESTED, + AMContainerState.STOPPING, AMContainerEventType.C_NM_STOP_FAILED, + new NMStopRequestFailedTransition()) + .addTransition(AMContainerState.STOP_REQUESTED, + AMContainerState.STOPPING, AMContainerEventType.C_NODE_FAILED, + new NodeFailedAtNMStopRequestedTransition()) + .addTransition( + AMContainerState.STOP_REQUESTED, + AMContainerState.STOP_REQUESTED, + EnumSet.of(AMContainerEventType.C_LAUNCHED, + AMContainerEventType.C_LAUNCH_FAILED, + AMContainerEventType.C_TA_SUCCEEDED, + AMContainerEventType.C_STOP_REQUEST, + AMContainerEventType.C_TIMED_OUT)) + .addTransition(AMContainerState.STOP_REQUESTED, + AMContainerState.STOP_REQUESTED, + AMContainerEventType.C_LAUNCH_REQUEST, + new ErrorAtNMStopRequestedTransition()) + + .addTransition(AMContainerState.STOPPING, AMContainerState.STOPPING, + AMContainerEventType.C_ASSIGN_TA, new AssignTAAtWindDownTransition()) + // TODO This transition is wrong. Should be a noop / error. + .addTransition(AMContainerState.STOPPING, AMContainerState.COMPLETED, + AMContainerEventType.C_COMPLETED, new CompletedAtWindDownTransition()) + .addTransition(AMContainerState.STOPPING, AMContainerState.STOPPING, + AMContainerEventType.C_NODE_FAILED, new NodeFailedBaseTransition()) + .addTransition( + AMContainerState.STOPPING, + AMContainerState.STOPPING, + EnumSet.of(AMContainerEventType.C_LAUNCHED, + AMContainerEventType.C_LAUNCH_FAILED, + AMContainerEventType.C_TA_SUCCEEDED, + AMContainerEventType.C_STOP_REQUEST, + AMContainerEventType.C_NM_STOP_SENT, + AMContainerEventType.C_NM_STOP_FAILED, + AMContainerEventType.C_TIMED_OUT)) + .addTransition(AMContainerState.STOPPING, AMContainerState.STOPPING, + AMContainerEventType.C_LAUNCH_REQUEST, + new ErrorAtStoppingTransition()) + + .addTransition(AMContainerState.COMPLETED, AMContainerState.COMPLETED, + AMContainerEventType.C_ASSIGN_TA, new AssignTAAtCompletedTransition()) + .addTransition(AMContainerState.COMPLETED, AMContainerState.COMPLETED, + AMContainerEventType.C_NODE_FAILED, new NodeFailedBaseTransition()) + .addTransition( + AMContainerState.COMPLETED, + AMContainerState.COMPLETED, + EnumSet.of(AMContainerEventType.C_LAUNCH_REQUEST, + AMContainerEventType.C_LAUNCHED, + AMContainerEventType.C_LAUNCH_FAILED, + AMContainerEventType.C_TA_SUCCEEDED, + AMContainerEventType.C_COMPLETED, + AMContainerEventType.C_STOP_REQUEST, + AMContainerEventType.C_NM_STOP_SENT, + AMContainerEventType.C_NM_STOP_FAILED, + AMContainerEventType.C_TIMED_OUT)) + + .installTopology(); // Note: Containers will not reach their final state if the RM link is broken, // AM shutdown should not wait for this. @@ -320,8 +318,8 @@ AMContainerEventType.C_NODE_FAILED, new NodeFailedBaseTransition()) // Attempting to use a container based purely on reosurces required, etc needs // additional change - JvmID, YarnChild, etc depend on TaskType. public AMContainerImpl(Container container, ContainerHeartbeatHandler chh, - TaskCommunicatorManagerInterface tal, ContainerSignatureMatcher signatureMatcher, - AppContext appContext, int schedulerId, int launcherId, int taskCommId, String auxiliaryService) { + TaskCommunicatorManagerInterface tal, ContainerSignatureMatcher signatureMatcher, + AppContext appContext, int schedulerId, int launcherId, int taskCommId, String auxiliaryService) { ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(); this.readLock = rwLock.readLock(); this.writeLock = rwLock.writeLock(); @@ -341,7 +339,6 @@ public AMContainerImpl(Container container, ContainerHeartbeatHandler chh, augmentStateMachine(); } - private void augmentStateMachine() { stateMachine .registerStateEnteredCallback(AMContainerState.STOP_REQUESTED, @@ -472,7 +469,7 @@ protected static class LaunchRequestTransition implements @Override public void transition(AMContainerImpl container, AMContainerEvent cEvent) { AMContainerEventLaunchRequest event = (AMContainerEventLaunchRequest) cEvent; - + ContainerContext containerContext = event.getContainerContext(); // Clone - don't use the object that is passed in, since this is likely to // be modified here. @@ -554,7 +551,7 @@ protected static class CompletedAtAllocatedTransition implements SingleArcTransition { @Override public void transition(AMContainerImpl container, AMContainerEvent cEvent) { - AMContainerEventCompleted event = (AMContainerEventCompleted)cEvent; + AMContainerEventCompleted event = (AMContainerEventCompleted) cEvent; String diag = event.getDiagnostics(); if (!(diag == null || diag.equals(""))) { LOG.info("Container " + container.getContainerId() @@ -588,7 +585,6 @@ public void transition(AMContainerImpl container, AMContainerEvent cEvent) { LOG.warn( "Unexpected event type: " + cEvent.getType() + " while in state: " + container.getState() + ". Event: " + cEvent); - } } @@ -610,7 +606,7 @@ public AMContainerState transition( container.handleExtraTAAssign(event, container.currentAttempt); return AMContainerState.STOP_REQUESTED; } - + Map taskLocalResources = event.getRemoteTaskLocalResources(); Preconditions.checkState(container.additionalLocalResources == null, "No additional resources should be pending when assigning a new task"); @@ -698,7 +694,7 @@ protected static class CompletedAtLaunchingTransition @Override public void transition(AMContainerImpl container, AMContainerEvent cEvent) { AMContainerEventCompleted event = (AMContainerEventCompleted) cEvent; - if (container.currentAttempt!= null) { + if (container.currentAttempt != null) { String errorMessage = getMessage(container, event); if (event.isSystemAction()) { container.sendContainerTerminatedBySystemToTaskAttempt(container.currentAttempt, @@ -709,7 +705,7 @@ public void transition(AMContainerImpl container, AMContainerEvent cEvent) { container.currentAttempt, errorMessage, // if termination cause is generic exited then replace with specific - (event.getTerminationCause() == TaskAttemptTerminationCause.CONTAINER_EXITED ? + (event.getTerminationCause() == TaskAttemptTerminationCause.CONTAINER_EXITED ? TaskAttemptTerminationCause.CONTAINER_LAUNCH_FAILED : event.getTerminationCause())); } container.registerFailedAttempt(container.currentAttempt); @@ -728,10 +724,10 @@ public void transition(AMContainerImpl container, AMContainerEvent cEvent) { } public String getMessage(AMContainerImpl container, - AMContainerEventCompleted event) { + AMContainerEventCompleted event) { return "Container" + container.getContainerId() + " finished while trying to launch. Diagnostics: [" - + event.getDiagnostics() +"]"; + + event.getDiagnostics() + "]"; } } @@ -746,7 +742,7 @@ public void transition(AMContainerImpl container, AMContainerEvent cEvent) { } container.unregisterFromTAListener(ContainerEndReason.OTHER, getMessage(container, cEvent)); container.logStopped(container.currentAttempt == null ? - ContainerExitStatus.SUCCESS + ContainerExitStatus.SUCCESS : ContainerExitStatus.INVALID); container.sendStopRequestToNM(); } @@ -927,7 +923,7 @@ public void transition(AMContainerImpl container, AMContainerEvent cEvent) { TezUtilsInternal.toTaskAttemptEndReason(event.getTerminationCause()), getMessage(container, event)); container.registerFailedAttempt(container.currentAttempt); - container.currentAttempt= null; + container.currentAttempt = null; super.transition(container, cEvent); } } @@ -1000,7 +996,7 @@ public void transition(AMContainerImpl container, AMContainerEvent cEvent) { AMContainerEventCompleted event = (AMContainerEventCompleted) cEvent; String diag = event.getDiagnostics(); for (TezTaskAttemptID taId : container.failedAssignments) { - container.sendTerminatedToTaskAttempt(taId, diag, + container.sendTerminatedToTaskAttempt(taId, diag, TaskAttemptTerminationCause.CONTAINER_EXITED); } if (container.currentAttempt != null) { @@ -1130,13 +1126,13 @@ private void logStopped(int exitStatus) { final Clock clock = appContext.getClock(); final HistoryEventHandler historyHandler = appContext.getHistoryHandler(); ContainerStoppedEvent lEvt = new ContainerStoppedEvent(containerId, - clock.getTime(), - exitStatus, + clock.getTime(), + exitStatus, appContext.getApplicationAttemptId()); historyHandler.handle( - new DAGHistoryEvent(appContext.getCurrentDAGID(),lEvt)); + new DAGHistoryEvent(appContext.getCurrentDAGID(), lEvt)); } - + protected void deAllocate() { sendEvent(new AMSchedulerEventDeallocateContainer(containerId, schedulerId)); } @@ -1145,14 +1141,14 @@ protected void sendTerminatedToTaskAttempt( TezTaskAttemptID taId, String message, TaskAttemptTerminationCause errCause) { sendEvent(new TaskAttemptEventContainerTerminated(containerId, taId, message, errCause)); } - + protected void sendContainerTerminatedBySystemToTaskAttempt( - TezTaskAttemptID taId, String message, TaskAttemptTerminationCause errorCause) { - sendEvent(new TaskAttemptEventContainerTerminatedBySystem(containerId, taId, message, errorCause)); + TezTaskAttemptID taId, String message, TaskAttemptTerminationCause errorCause) { + sendEvent(new TaskAttemptEventContainerTerminatedBySystem(containerId, taId, message, errorCause)); } protected void sendTerminatingToTaskAttempt(TezTaskAttemptID taId, - String message, TaskAttemptTerminationCause errorCause) { + String message, TaskAttemptTerminationCause errorCause) { sendEvent(new TaskAttemptEventContainerTerminating(taId, message, errorCause)); } @@ -1162,8 +1158,8 @@ protected void maybeSendNodeFailureForFailedAssignment(TezTaskAttemptID taId) { } } - protected void sendNodeFailureToTA(TezTaskAttemptID taId, String message, - TaskAttemptTerminationCause errorCause) { + protected void sendNodeFailureToTA(TezTaskAttemptID taId, String message, + TaskAttemptTerminationCause errorCause) { sendEvent(new TaskAttemptEventNodeFailed(taId, message, errorCause)); } @@ -1199,7 +1195,4 @@ protected void registerWithContainerListener() { protected void unregisterFromContainerListener() { this.containerHeartbeatHandler.unregister(this.containerId); } - - - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java index 15338e3332..2200557816 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerMap.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -50,7 +50,7 @@ public class AMContainerMap extends AbstractService implements EventHandler values() { return containerMap.values(); } - public void dagComplete(DAG dag){ + public void dagComplete(DAG dag) { AMContainerHelpers.dagComplete(dag.getID()); // Cleanup completed containers after a query completes. cleanupCompletedContainers(); @@ -117,5 +117,4 @@ private void cleanupCompletedContainers() { "Cleaned up completed containers on dagComplete. Removed={}, Remaining={}", count, containerMap.size()); } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerState.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerState.java index 7d58faac68..6a535bb365 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerState.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerState.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -24,7 +24,7 @@ public enum AMContainerState { RUNNING, // indicates a NM stop request has been attempted. This request could fail, in // which case an RM stop request needs to be sent. - STOP_REQUESTED, + STOP_REQUESTED, // A stop request has been registered with YARN STOPPING, diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerTask.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerTask.java index cfc143a402..35cbc3418a 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerTask.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerTask.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -50,11 +50,11 @@ public TaskSpec getTask() { public Map getAdditionalResources() { return this.additionalResources; } - + public Credentials getCredentials() { return this.credentials; } - + public boolean haveCredentialsChanged() { return this.credentialsChanged; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/ContainerContextMatcher.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/ContainerContextMatcher.java index f9c57c8843..dafe67f859 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/ContainerContextMatcher.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/ContainerContextMatcher.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.rm.container; @@ -35,13 +35,12 @@ private void checkArguments(Object cs1, Object cs2) { Objects.requireNonNull(cs1, "Arguments cannot be null"); Objects.requireNonNull(cs2, "Arguments cannot be null"); Preconditions.checkArgument(cs1 instanceof ContainerContext - && cs2 instanceof ContainerContext, + && cs2 instanceof ContainerContext, "Container context can only compare instances of " + ContainerContext.class.getName() + ", Recevied: " + cs1.getClass().getName() + " and " + cs2.getClass().getName()); - } - + @Override public boolean isSuperSet(Object cs1, Object cs2) { checkArguments(cs1, cs2); @@ -62,7 +61,7 @@ public boolean isExactMatch(Object cs1, Object cs2) { @Override public Map getAdditionalResources(Map lr1, - Map lr2) { + Map lr2) { Objects.requireNonNull(lr1); Objects.requireNonNull(lr2); @@ -78,5 +77,4 @@ public Object union(Object cs1, Object cs2) { checkArguments(cs1, cs2); return ContainerContext.union((ContainerContext) cs1, (ContainerContext) cs2); } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNode.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNode.java index bc01e04081..1cbd45ab84 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNode.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNode.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,13 +26,17 @@ import org.apache.tez.dag.app.dag.DAG; public interface AMNode extends EventHandler { - + public NodeId getNodeId(); + public AMNodeState getState(); + public List getContainers(); public boolean isUnhealthy(); + public boolean isBlacklisted(); + public boolean isUsable(); void dagComplete(DAG dag); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEvent.java index d9e249af4e..012d250816 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventContainerAllocated.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventContainerAllocated.java index e250f42d3a..0b48273cfd 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventContainerAllocated.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventContainerAllocated.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -32,5 +32,4 @@ public AMNodeEventContainerAllocated(NodeId nodeId, int sourceId, ContainerId co public ContainerId getContainerId() { return this.containerId; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventContainerCompleted.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventContainerCompleted.java index f999c3aef2..6496dc5723 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventContainerCompleted.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventContainerCompleted.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventNodeCountUpdated.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventNodeCountUpdated.java index 3b35daf2a8..db2a1a1cfc 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventNodeCountUpdated.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventNodeCountUpdated.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,12 +21,12 @@ public class AMNodeEventNodeCountUpdated extends AMNodeEvent { private final int count; - + public AMNodeEventNodeCountUpdated(int nodeCount, int sourceId) { super(null, sourceId, AMNodeEventType.N_NODE_COUNT_UPDATED); this.count = nodeCount; } - + public int getNodeCount() { return this.count; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventStateChanged.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventStateChanged.java index b371ddde05..2b0d156010 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventStateChanged.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventStateChanged.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -25,14 +25,13 @@ public class AMNodeEventStateChanged extends AMNodeEvent { public AMNodeEventStateChanged(NodeReport nodeReport, int sourceId) { super(nodeReport.getNodeId(), sourceId, - (nodeReport.getNodeState().isUnusable() ? - AMNodeEventType.N_TURNED_UNHEALTHY : - AMNodeEventType.N_TURNED_HEALTHY)); + (nodeReport.getNodeState().isUnusable() ? + AMNodeEventType.N_TURNED_UNHEALTHY : + AMNodeEventType.N_TURNED_HEALTHY)); this.nodeReport = nodeReport; } public NodeReport getNodeReport() { return this.nodeReport; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java index 4a4cb6107a..75e60be73e 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptEnded.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -26,9 +26,9 @@ public class AMNodeEventTaskAttemptEnded extends AMNodeEvent { private final boolean failed; private final ContainerId containerId; private final TezTaskAttemptID taskAttemptId; - + public AMNodeEventTaskAttemptEnded(NodeId nodeId, int sourceId, ContainerId containerId, - TezTaskAttemptID taskAttemptId, boolean failed) { + TezTaskAttemptID taskAttemptId, boolean failed) { super(nodeId, sourceId, AMNodeEventType.N_TA_ENDED); this.failed = failed; this.containerId = containerId; @@ -38,15 +38,15 @@ public AMNodeEventTaskAttemptEnded(NodeId nodeId, int sourceId, ContainerId cont public boolean failed() { return failed; } - + public boolean killed() { return !failed; } - + public ContainerId getContainerId() { return this.containerId; } - + public TezTaskAttemptID getTaskAttemptId() { return this.taskAttemptId; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptSucceeded.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptSucceeded.java index 2b8cb7df8e..caa86223d8 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptSucceeded.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventTaskAttemptSucceeded.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -28,7 +28,7 @@ public class AMNodeEventTaskAttemptSucceeded extends AMNodeEvent { private final TezTaskAttemptID taskAttemptId; public AMNodeEventTaskAttemptSucceeded(NodeId nodeId, int sourceId, - ContainerId containerId, TezTaskAttemptID taskAttemptId) { + ContainerId containerId, TezTaskAttemptID taskAttemptId) { super(nodeId, sourceId, AMNodeEventType.N_TA_SUCCEEDED); this.containerId = containerId; this.taskAttemptId = taskAttemptId; @@ -41,5 +41,4 @@ public ContainerId getContainerId() { public TezTaskAttemptID getTaskAttemptId() { return this.taskAttemptId; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventType.java index a14112486a..74ec0ae3cd 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventType.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeEventType.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.rm.node; @@ -30,14 +30,13 @@ public enum AMNodeEventType { // Producer: TaskSchedulerEventHnadler, Task(retroactive failure) N_TA_ENDED, - + //Producer: TaskScheduler via TaskSchedulerEventHandler N_TURNED_UNHEALTHY, N_TURNED_HEALTHY, N_NODE_COUNT_UPDATED, // for blacklisting. - + //Producer: AMNodeManager N_IGNORE_BLACKLISTING_ENABLED, N_IGNORE_BLACKLISTING_DISABLED, - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeImpl.java index 26796d2af4..9fa53a772e 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeImpl.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeImpl.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.rm.node; @@ -80,113 +80,112 @@ public class AMNodeImpl implements AMNode { private final StateMachine stateMachine; private static StateMachineFactory - - stateMachineFactory = - new StateMachineFactory( - AMNodeState.ACTIVE) - // Transitions from ACTIVE state. - .addTransition(AMNodeState.ACTIVE, AMNodeState.ACTIVE, - AMNodeEventType.N_CONTAINER_ALLOCATED, - new ContainerAllocatedTransition()) - .addTransition(AMNodeState.ACTIVE, AMNodeState.ACTIVE, - AMNodeEventType.N_TA_SUCCEEDED, new TaskAttemptSucceededTransition()) - .addTransition(AMNodeState.ACTIVE, - EnumSet.of(AMNodeState.ACTIVE, AMNodeState.BLACKLISTED), - AMNodeEventType.N_TA_ENDED, new TaskAttemptFailedTransition()) - .addTransition(AMNodeState.ACTIVE, AMNodeState.UNHEALTHY, - AMNodeEventType.N_TURNED_UNHEALTHY, - new NodeTurnedUnhealthyTransition()) - .addTransition(AMNodeState.ACTIVE, - EnumSet.of(AMNodeState.ACTIVE, AMNodeState.BLACKLISTED), - AMNodeEventType.N_IGNORE_BLACKLISTING_DISABLED, - new IgnoreBlacklistingDisabledTransition()) - .addTransition(AMNodeState.ACTIVE, AMNodeState.FORCED_ACTIVE, - AMNodeEventType.N_IGNORE_BLACKLISTING_ENABLED, - new IgnoreBlacklistingStateChangeTransition(true)) - .addTransition(AMNodeState.ACTIVE, AMNodeState.ACTIVE, - AMNodeEventType.N_TURNED_HEALTHY) - .addTransition(AMNodeState.ACTIVE, AMNodeState.ACTIVE, - AMNodeEventType.N_CONTAINER_COMPLETED, CONTAINER_COMPLETED_TRANSITION) - - // Transitions from BLACKLISTED state. - .addTransition(AMNodeState.BLACKLISTED, AMNodeState.BLACKLISTED, - AMNodeEventType.N_CONTAINER_ALLOCATED, - new ContainerAllocatedWhileBlacklistedTransition()) - .addTransition(AMNodeState.BLACKLISTED, - EnumSet.of(AMNodeState.BLACKLISTED, AMNodeState.ACTIVE), - AMNodeEventType.N_TA_SUCCEEDED, - new TaskAttemptSucceededWhileBlacklistedTransition()) - .addTransition(AMNodeState.BLACKLISTED, AMNodeState.BLACKLISTED, - AMNodeEventType.N_TA_ENDED, new CountFailedTaskAttemptTransition()) - .addTransition(AMNodeState.BLACKLISTED, AMNodeState.UNHEALTHY, - AMNodeEventType.N_TURNED_UNHEALTHY, - new NodeTurnedUnhealthyTransition()) - .addTransition(AMNodeState.BLACKLISTED, AMNodeState.FORCED_ACTIVE, - AMNodeEventType.N_IGNORE_BLACKLISTING_ENABLED, - new IgnoreBlacklistingStateChangeTransition(true)) - .addTransition(AMNodeState.BLACKLISTED, AMNodeState.BLACKLISTED, - AMNodeEventType.N_CONTAINER_COMPLETED, CONTAINER_COMPLETED_TRANSITION) - .addTransition( - AMNodeState.BLACKLISTED, - AMNodeState.BLACKLISTED, - EnumSet.of(AMNodeEventType.N_TURNED_HEALTHY, - AMNodeEventType.N_IGNORE_BLACKLISTING_DISABLED), - new GenericErrorTransition()) - - // Transitions from FORCED_ACTIVE state. - .addTransition(AMNodeState.FORCED_ACTIVE, AMNodeState.FORCED_ACTIVE, - AMNodeEventType.N_CONTAINER_ALLOCATED, - new ContainerAllocatedTransition()) - .addTransition(AMNodeState.FORCED_ACTIVE, AMNodeState.FORCED_ACTIVE, - AMNodeEventType.N_TA_SUCCEEDED, new TaskAttemptSucceededTransition()) - .addTransition(AMNodeState.FORCED_ACTIVE, AMNodeState.FORCED_ACTIVE, - AMNodeEventType.N_TA_ENDED, new CountFailedTaskAttemptTransition()) - .addTransition(AMNodeState.FORCED_ACTIVE, AMNodeState.UNHEALTHY, - AMNodeEventType.N_TURNED_UNHEALTHY, - new NodeTurnedUnhealthyTransition()) - .addTransition(AMNodeState.FORCED_ACTIVE, - EnumSet.of(AMNodeState.BLACKLISTED, AMNodeState.ACTIVE), - AMNodeEventType.N_IGNORE_BLACKLISTING_DISABLED, - new IgnoreBlacklistingDisabledTransition()) - .addTransition(AMNodeState.FORCED_ACTIVE, AMNodeState.FORCED_ACTIVE, - AMNodeEventType.N_CONTAINER_COMPLETED, CONTAINER_COMPLETED_TRANSITION) - .addTransition( - AMNodeState.FORCED_ACTIVE, - AMNodeState.FORCED_ACTIVE, - EnumSet.of(AMNodeEventType.N_TURNED_HEALTHY, - AMNodeEventType.N_IGNORE_BLACKLISTING_ENABLED), - new GenericErrorTransition()) - - // Transitions from UNHEALTHY state. - .addTransition(AMNodeState.UNHEALTHY, AMNodeState.UNHEALTHY, - AMNodeEventType.N_CONTAINER_ALLOCATED, - new ContainerAllocatedWhileUnhealthyTransition()) - .addTransition( - AMNodeState.UNHEALTHY, - AMNodeState.UNHEALTHY, - EnumSet - .of(AMNodeEventType.N_TA_SUCCEEDED, AMNodeEventType.N_TA_ENDED)) - .addTransition(AMNodeState.UNHEALTHY, AMNodeState.UNHEALTHY, - AMNodeEventType.N_IGNORE_BLACKLISTING_DISABLED, - new IgnoreBlacklistingStateChangeTransition(false)) - .addTransition(AMNodeState.UNHEALTHY, AMNodeState.UNHEALTHY, - AMNodeEventType.N_IGNORE_BLACKLISTING_ENABLED, - new IgnoreBlacklistingStateChangeTransition(true)) - .addTransition(AMNodeState.UNHEALTHY, - EnumSet.of(AMNodeState.ACTIVE, AMNodeState.FORCED_ACTIVE), - AMNodeEventType.N_TURNED_HEALTHY, new NodeTurnedHealthyTransition()) - .addTransition(AMNodeState.UNHEALTHY, AMNodeState.UNHEALTHY, - AMNodeEventType.N_CONTAINER_COMPLETED, CONTAINER_COMPLETED_TRANSITION) - .addTransition(AMNodeState.UNHEALTHY, AMNodeState.UNHEALTHY, - AMNodeEventType.N_TURNED_UNHEALTHY, new GenericErrorTransition()) - - .installTopology(); - + + stateMachineFactory = + new StateMachineFactory( + AMNodeState.ACTIVE) + // Transitions from ACTIVE state. + .addTransition(AMNodeState.ACTIVE, AMNodeState.ACTIVE, + AMNodeEventType.N_CONTAINER_ALLOCATED, + new ContainerAllocatedTransition()) + .addTransition(AMNodeState.ACTIVE, AMNodeState.ACTIVE, + AMNodeEventType.N_TA_SUCCEEDED, new TaskAttemptSucceededTransition()) + .addTransition(AMNodeState.ACTIVE, + EnumSet.of(AMNodeState.ACTIVE, AMNodeState.BLACKLISTED), + AMNodeEventType.N_TA_ENDED, new TaskAttemptFailedTransition()) + .addTransition(AMNodeState.ACTIVE, AMNodeState.UNHEALTHY, + AMNodeEventType.N_TURNED_UNHEALTHY, + new NodeTurnedUnhealthyTransition()) + .addTransition(AMNodeState.ACTIVE, + EnumSet.of(AMNodeState.ACTIVE, AMNodeState.BLACKLISTED), + AMNodeEventType.N_IGNORE_BLACKLISTING_DISABLED, + new IgnoreBlacklistingDisabledTransition()) + .addTransition(AMNodeState.ACTIVE, AMNodeState.FORCED_ACTIVE, + AMNodeEventType.N_IGNORE_BLACKLISTING_ENABLED, + new IgnoreBlacklistingStateChangeTransition(true)) + .addTransition(AMNodeState.ACTIVE, AMNodeState.ACTIVE, + AMNodeEventType.N_TURNED_HEALTHY) + .addTransition(AMNodeState.ACTIVE, AMNodeState.ACTIVE, + AMNodeEventType.N_CONTAINER_COMPLETED, CONTAINER_COMPLETED_TRANSITION) + + // Transitions from BLACKLISTED state. + .addTransition(AMNodeState.BLACKLISTED, AMNodeState.BLACKLISTED, + AMNodeEventType.N_CONTAINER_ALLOCATED, + new ContainerAllocatedWhileBlacklistedTransition()) + .addTransition(AMNodeState.BLACKLISTED, + EnumSet.of(AMNodeState.BLACKLISTED, AMNodeState.ACTIVE), + AMNodeEventType.N_TA_SUCCEEDED, + new TaskAttemptSucceededWhileBlacklistedTransition()) + .addTransition(AMNodeState.BLACKLISTED, AMNodeState.BLACKLISTED, + AMNodeEventType.N_TA_ENDED, new CountFailedTaskAttemptTransition()) + .addTransition(AMNodeState.BLACKLISTED, AMNodeState.UNHEALTHY, + AMNodeEventType.N_TURNED_UNHEALTHY, + new NodeTurnedUnhealthyTransition()) + .addTransition(AMNodeState.BLACKLISTED, AMNodeState.FORCED_ACTIVE, + AMNodeEventType.N_IGNORE_BLACKLISTING_ENABLED, + new IgnoreBlacklistingStateChangeTransition(true)) + .addTransition(AMNodeState.BLACKLISTED, AMNodeState.BLACKLISTED, + AMNodeEventType.N_CONTAINER_COMPLETED, CONTAINER_COMPLETED_TRANSITION) + .addTransition( + AMNodeState.BLACKLISTED, + AMNodeState.BLACKLISTED, + EnumSet.of(AMNodeEventType.N_TURNED_HEALTHY, + AMNodeEventType.N_IGNORE_BLACKLISTING_DISABLED), + new GenericErrorTransition()) + + // Transitions from FORCED_ACTIVE state. + .addTransition(AMNodeState.FORCED_ACTIVE, AMNodeState.FORCED_ACTIVE, + AMNodeEventType.N_CONTAINER_ALLOCATED, + new ContainerAllocatedTransition()) + .addTransition(AMNodeState.FORCED_ACTIVE, AMNodeState.FORCED_ACTIVE, + AMNodeEventType.N_TA_SUCCEEDED, new TaskAttemptSucceededTransition()) + .addTransition(AMNodeState.FORCED_ACTIVE, AMNodeState.FORCED_ACTIVE, + AMNodeEventType.N_TA_ENDED, new CountFailedTaskAttemptTransition()) + .addTransition(AMNodeState.FORCED_ACTIVE, AMNodeState.UNHEALTHY, + AMNodeEventType.N_TURNED_UNHEALTHY, + new NodeTurnedUnhealthyTransition()) + .addTransition(AMNodeState.FORCED_ACTIVE, + EnumSet.of(AMNodeState.BLACKLISTED, AMNodeState.ACTIVE), + AMNodeEventType.N_IGNORE_BLACKLISTING_DISABLED, + new IgnoreBlacklistingDisabledTransition()) + .addTransition(AMNodeState.FORCED_ACTIVE, AMNodeState.FORCED_ACTIVE, + AMNodeEventType.N_CONTAINER_COMPLETED, CONTAINER_COMPLETED_TRANSITION) + .addTransition( + AMNodeState.FORCED_ACTIVE, + AMNodeState.FORCED_ACTIVE, + EnumSet.of(AMNodeEventType.N_TURNED_HEALTHY, + AMNodeEventType.N_IGNORE_BLACKLISTING_ENABLED), + new GenericErrorTransition()) + + // Transitions from UNHEALTHY state. + .addTransition(AMNodeState.UNHEALTHY, AMNodeState.UNHEALTHY, + AMNodeEventType.N_CONTAINER_ALLOCATED, + new ContainerAllocatedWhileUnhealthyTransition()) + .addTransition( + AMNodeState.UNHEALTHY, + AMNodeState.UNHEALTHY, + EnumSet + .of(AMNodeEventType.N_TA_SUCCEEDED, AMNodeEventType.N_TA_ENDED)) + .addTransition(AMNodeState.UNHEALTHY, AMNodeState.UNHEALTHY, + AMNodeEventType.N_IGNORE_BLACKLISTING_DISABLED, + new IgnoreBlacklistingStateChangeTransition(false)) + .addTransition(AMNodeState.UNHEALTHY, AMNodeState.UNHEALTHY, + AMNodeEventType.N_IGNORE_BLACKLISTING_ENABLED, + new IgnoreBlacklistingStateChangeTransition(true)) + .addTransition(AMNodeState.UNHEALTHY, + EnumSet.of(AMNodeState.ACTIVE, AMNodeState.FORCED_ACTIVE), + AMNodeEventType.N_TURNED_HEALTHY, new NodeTurnedHealthyTransition()) + .addTransition(AMNodeState.UNHEALTHY, AMNodeState.UNHEALTHY, + AMNodeEventType.N_CONTAINER_COMPLETED, CONTAINER_COMPLETED_TRANSITION) + .addTransition(AMNodeState.UNHEALTHY, AMNodeState.UNHEALTHY, + AMNodeEventType.N_TURNED_UNHEALTHY, new GenericErrorTransition()) + + .installTopology(); @SuppressWarnings("rawtypes") public AMNodeImpl(NodeId nodeId, int schedulerId, int maxTaskFailuresPerNode, - EventHandler eventHandler, boolean blacklistingEnabled, - boolean rescheduleOnUnhealthyNode, AppContext appContext) { + EventHandler eventHandler, boolean blacklistingEnabled, + boolean rescheduleOnUnhealthyNode, AppContext appContext) { ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(); this.readLock = rwLock.readLock(); this.writeLock = rwLock.writeLock(); @@ -480,7 +479,7 @@ public boolean isBlacklisted() { this.readLock.unlock(); } } - + @Override public boolean isUsable() { return !(isUnhealthy() || isBlacklisted()); @@ -493,7 +492,6 @@ public void dagComplete(DAG dag) { int countBefore = containers.size(); int countCompleted = completedContainers.size(); - // Actual functionality. containers.removeAll(completedContainers); completedContainers.clear(); @@ -504,7 +502,6 @@ public void dagComplete(DAG dag) { "Node {}, cleaning up knownContainers. current={}, completed={}, postCleanup={}", getNodeId(), countBefore, countCompleted, countAfter); } - } finally { this.writeLock.unlock(); } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeState.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeState.java index 380d18e25f..82c9632abd 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeState.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeState.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeTracker.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeTracker.java index 8c81cb52c9..5fb289ebbd 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeTracker.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/AMNodeTracker.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.rm.node; @@ -37,9 +37,9 @@ public class AMNodeTracker extends AbstractService implements EventHandler { - + static final Logger LOG = LoggerFactory.getLogger(AMNodeTracker.class); - + private final ConcurrentMap perSourceNodeTrackers; @SuppressWarnings("rawtypes") @@ -59,21 +59,21 @@ public AMNodeTracker(EventHandler eventHandler, AppContext appContext) { this.eventHandler = eventHandler; this.appContext = appContext; } - + @Override public synchronized void serviceInit(Configuration conf) { this.maxTaskFailuresPerNode = conf.getInt( - TezConfiguration.TEZ_AM_MAX_TASK_FAILURES_PER_NODE, + TezConfiguration.TEZ_AM_MAX_TASK_FAILURES_PER_NODE, TezConfiguration.TEZ_AM_MAX_TASK_FAILURES_PER_NODE_DEFAULT); this.nodeBlacklistingEnabled = conf.getBoolean( TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED_DEFAULT); this.blacklistDisablePercent = conf.getInt( - TezConfiguration.TEZ_AM_NODE_BLACKLISTING_IGNORE_THRESHOLD, - TezConfiguration.TEZ_AM_NODE_BLACKLISTING_IGNORE_THRESHOLD_DEFAULT); + TezConfiguration.TEZ_AM_NODE_BLACKLISTING_IGNORE_THRESHOLD, + TezConfiguration.TEZ_AM_NODE_BLACKLISTING_IGNORE_THRESHOLD_DEFAULT); this.nodeUpdatesRescheduleEnabled = conf.getBoolean( - TezConfiguration.TEZ_AM_NODE_UNHEALTHY_RESCHEDULE_TASKS, - TezConfiguration.TEZ_AM_NODE_UNHEALTHY_RESCHEDULE_TASKS_DEFAULT); + TezConfiguration.TEZ_AM_NODE_UNHEALTHY_RESCHEDULE_TASKS, + TezConfiguration.TEZ_AM_NODE_UNHEALTHY_RESCHEDULE_TASKS_DEFAULT); LOG.info("blacklistDisablePercent is " + blacklistDisablePercent + ", blacklistingEnabled: " + nodeBlacklistingEnabled + @@ -92,7 +92,6 @@ public void nodeSeen(NodeId nodeId, int schedulerId) { nodeTracker.nodeSeen(nodeId); } - boolean registerBadNodeAndShouldBlacklist(AMNode amNode, int schedulerId) { return perSourceNodeTrackers.get(schedulerId).registerBadNodeAndShouldBlacklist(amNode); } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/PerSourceNodeTracker.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/PerSourceNodeTracker.java index 9906644fe1..2d4d201554 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/PerSourceNodeTracker.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/node/PerSourceNodeTracker.java @@ -66,8 +66,6 @@ public PerSourceNodeTracker(int sourceId, EventHandler eventHandler, AppContext this.nodeUpdatesRescheduleEnabled = nodeUpdatesRescheduleEnabled; } - - public void nodeSeen(NodeId nodeId) { if (nodeMap.putIfAbsent(nodeId, new AMNodeImpl(nodeId, sourceId, maxTaskFailuresPerNode, eventHandler, nodeBlacklistingEnabled, nodeUpdatesRescheduleEnabled, diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/package-info.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/package-info.java index b41a53a203..c6cb12b83e 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/package-info.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/package-info.java @@ -17,4 +17,5 @@ */ @InterfaceAudience.Private package org.apache.tez.dag.app.rm; + import org.apache.hadoop.classification.InterfaceAudience; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/security/authorize/TezAMPolicyProvider.java b/tez-dag/src/main/java/org/apache/tez/dag/app/security/authorize/TezAMPolicyProvider.java index a212041183..28bd9d5cc0 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/security/authorize/TezAMPolicyProvider.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/security/authorize/TezAMPolicyProvider.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,20 +31,19 @@ @InterfaceAudience.Private @InterfaceStability.Unstable public class TezAMPolicyProvider extends PolicyProvider { - - private static final Service[] tezApplicationMasterServices = - new Service[] { - new Service( - TezConstants.TEZ_AM_SECURITY_SERVICE_AUTHORIZATION_TASK_UMBILICAL, - TezTaskUmbilicalProtocol.class), - new Service( - TezConstants.TEZ_AM_SECURITY_SERVICE_AUTHORIZATION_CLIENT, - DAGClientAMProtocolBlockingPB.class) - }; + + private static final Service[] tezApplicationMasterServices = + new Service[]{ + new Service( + TezConstants.TEZ_AM_SECURITY_SERVICE_AUTHORIZATION_TASK_UMBILICAL, + TezTaskUmbilicalProtocol.class), + new Service( + TezConstants.TEZ_AM_SECURITY_SERVICE_AUTHORIZATION_CLIENT, + DAGClientAMProtocolBlockingPB.class) + }; @Override public Service[] getServices() { return tezApplicationMasterServices.clone(); } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/security/authorize/package-info.java b/tez-dag/src/main/java/org/apache/tez/dag/app/security/authorize/package-info.java index 7935003abe..76b90f0695 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/security/authorize/package-info.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/security/authorize/package-info.java @@ -17,4 +17,5 @@ */ @InterfaceAudience.Private package org.apache.tez.dag.app.security.authorize; + import org.apache.hadoop.classification.InterfaceAudience; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/web/AMWebController.java b/tez-dag/src/main/java/org/apache/tez/dag/app/web/AMWebController.java index 00cd26e2ce..d2165e5c8d 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/web/AMWebController.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/web/AMWebController.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -139,7 +139,7 @@ public void setCorsHeaders() { */ String historyUrlBase = appContext.getAMConf().get(TezConfiguration.TEZ_HISTORY_URL_BASE, ""); String origin = request().getHeader(ORIGIN); - if(origin == null) { + if (origin == null) { try { URL url = new URL(historyUrlBase); origin = url.getProtocol() + "://" + url.getAuthority(); @@ -259,7 +259,6 @@ public void getVertexProgress() { renderJSON(result); } - Collection getVerticesByIdx(DAG dag, Collection indexes) { Collection vertices = new ArrayList(indexes.size()); final TezDAGID tezDAGID = dag.getID(); @@ -275,7 +274,7 @@ Collection getVerticesByIdx(DAG dag, Collection indexes) { } } - return vertices; + return vertices; } int getQueryParamInt(String name) throws NumberFormatException { @@ -319,7 +318,7 @@ public void getVertexProgresses() { } Collection progresses = new ArrayList(vertices.size()); - for(Vertex vertex : vertices) { + for (Vertex vertex : vertices) { progresses.add(new ProgressInfo(vertex.getVertexId().toString(), vertex.getCompletedTaskProgress())); } @@ -418,7 +417,7 @@ Map> getCounterListFromRequest() { String counterGroup = token.substring(0, pos); Set counters = Collections.emptySet(); if (pos < token.length() - 1) { - String counterNames = token.substring(pos+1, token.length()); + String counterNames = token.substring(pos + 1, token.length()); counters = Sets.newHashSet( Splitter.on(counterDelimiter).omitEmptyStrings() .trimResults().split(counterNames)); @@ -428,12 +427,11 @@ Map> getCounterListFromRequest() { return counterList; } - List splitString(String str, String delimiter, Integer limit) { List items = new ArrayList(); StringTokenizer tokenizer = new StringTokenizer(str, delimiter); - for(int count = 0; tokenizer.hasMoreElements() && count < limit; count ++) { + for (int count = 0; tokenizer.hasMoreElements() && count < limit; count++) { items.add(tokenizer.nextToken()); } @@ -491,7 +489,7 @@ List> getIDsFromRequest(String paramName, Integer limit, Integer c for (String valueStr : splitString(valuesStr, ",", limit)) { List innerValues = new ArrayList(); String innerValueStrs[] = valueStr.split("_"); - if(innerValueStrs.length == count) { + if (innerValueStrs.length == count) { for (String innerValueStr : innerValueStrs) { int value = Integer.parseInt(innerValueStr); innerValues.add(value); @@ -544,7 +542,7 @@ public void getDagInfo() { } Map> constructCounterMapInfo(TezCounters counters, - Map> counterNames) { + Map> counterNames) { if (counterNames == null || counterNames.isEmpty()) { return null; } @@ -645,7 +643,7 @@ public void getVerticesInfo() { } ArrayList> verticesInfo = new ArrayList>(); - for(Vertex v : vertexList) { + for (Vertex v : vertexList) { verticesInfo.add(getVertexInfoMap(v, counterNames)); } @@ -672,54 +670,49 @@ List getRequestedTasks(DAG dag, Integer limit) { List tasks = new ArrayList(); List> taskIDs = getIDsFromRequest(WebUIService.TASK_ID, limit, 2); - if(taskIDs == null) { + if (taskIDs == null) { return null; - } - else if(!taskIDs.isEmpty()) { + } else if (!taskIDs.isEmpty()) { for (List indexes : taskIDs) { Vertex vertex = getVertexFromIndex(dag, indexes.get(0)); - if(vertex == null) { + if (vertex == null) { continue; } Task task = vertex.getTask(indexes.get(1)); - if(task == null) { + if (task == null) { continue; - } - else { + } else { tasks.add(task); } - if(tasks.size() >= limit) { + if (tasks.size() >= limit) { break; } } - } - else { + } else { List vertexIDs = getIntegersFromRequest(WebUIService.VERTEX_ID, limit); - if(vertexIDs == null) { + if (vertexIDs == null) { return null; - } - else if(!vertexIDs.isEmpty()) { + } else if (!vertexIDs.isEmpty()) { for (Integer vertexID : vertexIDs) { Vertex vertex = getVertexFromIndex(dag, vertexID); - if(vertex == null) { + if (vertex == null) { continue; } List vertexTasks = new ArrayList(vertex.getTasks().values()); tasks.addAll(vertexTasks.subList(0, Math.min(vertexTasks.size(), limit - tasks.size()))); - if(tasks.size() >= limit) { + if (tasks.size() >= limit) { break; } } - } - else { + } else { Collection vertices = dag.getVertices().values(); for (Vertex vertex : vertices) { List vertexTasks = new ArrayList(vertex.getTasks().values()); tasks.addAll(vertexTasks.subList(0, Math.min(vertexTasks.size(), limit - tasks.size()))); - if(tasks.size() >= limit) { + if (tasks.size() >= limit) { break; } } @@ -751,14 +744,14 @@ public void getTasksInfo() { } List tasks = getRequestedTasks(dag, limit); - if(tasks == null) { + if (tasks == null) { return; } Map> counterNames = getCounterListFromRequest(); ArrayList> tasksInfo = new ArrayList>(); - for(Task t : tasks) { + for (Task t : tasks) { Map taskInfo = new HashMap(); taskInfo.put("id", t.getTaskID().toString()); taskInfo.put("progress", Float.toString(t.getProgress())); @@ -779,7 +772,7 @@ public void getTasksInfo() { } renderJSON(ImmutableMap.of( - "tasks", tasksInfo + "tasks", tasksInfo )); } @@ -795,30 +788,28 @@ List getRequestedAttempts(DAG dag, Integer limit) { List attempts = new ArrayList(); List> attemptIDs = getIDsFromRequest(WebUIService.ATTEMPT_ID, limit, 3); - if(attemptIDs == null) { + if (attemptIDs == null) { return null; - } - else if(!attemptIDs.isEmpty()) { + } else if (!attemptIDs.isEmpty()) { for (List indexes : attemptIDs) { Vertex vertex = getVertexFromIndex(dag, indexes.get(0)); - if(vertex == null) { + if (vertex == null) { continue; } Task task = vertex.getTask(indexes.get(1)); - if(task == null) { + if (task == null) { continue; } TaskAttempt attempt = task. getAttempt(TezTaskAttemptID.getInstance(task.getTaskID(), indexes.get(2))); - if(attempt == null) { + if (attempt == null) { continue; - } - else { + } else { attempts.add(attempt); } - if(attempts.size() >= limit) { + if (attempts.size() >= limit) { break; } } @@ -849,14 +840,14 @@ public void getAttemptsInfo() { } List attempts = getRequestedAttempts(dag, limit); - if(attempts == null) { + if (attempts == null) { return; } Map> counterNames = getCounterListFromRequest(); ArrayList> attemptsInfo = new ArrayList>(); - for(TaskAttempt a : attempts) { + for (TaskAttempt a : attempts) { Map attemptInfo = new HashMap(); attemptInfo.put("id", a.getTaskAttemptID().toString()); attemptInfo.put("progress", Float.toString(a.getProgress())); @@ -888,7 +879,8 @@ public static class StaticAMView extends View { @Inject AppContext appContext; @Inject - @Named("TezUIHistoryURL") String historyUrl; + @Named("TezUIHistoryURL") + String historyUrl; @Override public void render() { @@ -909,8 +901,8 @@ public void render() { "here

    " ); pw.write(""); + "window.location.replace('" + historyUrl + "');" + + "}, 0); "); } pw.write(""); pw.write(""); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/web/WebUIService.java b/tez-dag/src/main/java/org/apache/tez/dag/app/web/WebUIService.java index bf94a73002..1aa383b909 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/web/WebUIService.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/web/WebUIService.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -119,7 +119,7 @@ protected void serviceStart() throws Exception { + context.getAppMaster().getAppNMHost()); } baseUrl = "http://" + hostname + ":" + port; - trackingUrl = baseUrl + "/ui/"; + trackingUrl = baseUrl + "/ui/"; LOG.info("Instantiated WebUIService at " + trackingUrl); } } catch (Exception e) { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/DAGHistoryEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/DAGHistoryEvent.java index 240d8bd96d..a2087fcee8 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/DAGHistoryEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/DAGHistoryEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -27,7 +27,7 @@ public class DAGHistoryEvent implements DAGIDAware { private final TezDAGID dagID; public DAGHistoryEvent(TezDAGID dagID, - HistoryEvent historyEvent) { + HistoryEvent historyEvent) { this.dagID = dagID; this.historyEvent = historyEvent; } @@ -44,5 +44,4 @@ public HistoryEvent getHistoryEvent() { public TezDAGID getDAGID() { return this.dagID; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEvent.java index 5b077e98ec..131822e918 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,5 +34,4 @@ public interface HistoryEvent { public void toProtoStream(CodedOutputStream outputStream) throws IOException; public void fromProtoStream(CodedInputStream inputStream) throws IOException; - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEventHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEventHandler.java index f4dd789fd9..526e571bae 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEventHandler.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEventHandler.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -91,7 +91,7 @@ public void serviceInit(Configuration conf) throws Exception { String recoveryServiceClass = conf.get(TezConfiguration.TEZ_AM_RECOVERY_SERVICE_CLASS, TezConfiguration.TEZ_AM_RECOVERY_SERVICE_CLASS_DEFAULT); recoveryService = ReflectionUtils.createClazzInstance(recoveryServiceClass, - new Class[]{AppContext.class}, new Object[] {context}); + new Class[]{AppContext.class}, new Object[]{context}); addService(recoveryService); } @@ -126,7 +126,7 @@ public void serviceStop() throws Exception { public void handleCriticalEvent(DAGHistoryEvent event) throws IOException { TezDAGID dagId = event.getDAGID(); String dagIdStr = "N/A"; - if(dagId != null) { + if (dagId != null) { dagIdStr = dagId.toString(); } HistoryEvent historyEvent = event.getHistoryEvent(); @@ -174,7 +174,7 @@ private boolean shouldLogEvent(DAGHistoryEvent event) { HistoryEvent historyEvent = event.getHistoryEvent(); HistoryEventType eventType = historyEvent.getEventType(); if (eventType == HistoryEventType.DAG_SUBMITTED) { - Configuration dagConf = ((DAGSubmittedEvent)historyEvent).getConf(); + Configuration dagConf = ((DAGSubmittedEvent) historyEvent).getConf(); dagLogLevel = HistoryLogLevel.getLogLevel(dagConf, amHistoryLogLevel); dagIdToLogLevel.put(dagId, dagLogLevel); maybeUpdateDagTaskAttemptFilters(dagId, dagLogLevel, dagConf); @@ -206,7 +206,7 @@ private boolean shouldLogTaskAttemptEvents(DAGHistoryEvent event, HistoryLogLeve HistoryEventType eventType = historyEvent.getEventType(); if (dagLogLevel == HistoryLogLevel.TASK_ATTEMPT && (eventType == HistoryEventType.TASK_ATTEMPT_STARTED || - eventType == HistoryEventType.TASK_ATTEMPT_FINISHED)) { + eventType == HistoryEventType.TASK_ATTEMPT_FINISHED)) { TezDAGID dagId = event.getDAGID(); Set filters = null; if (dagId != null) { @@ -219,10 +219,10 @@ private boolean shouldLogTaskAttemptEvents(DAGHistoryEvent event, HistoryLogLeve return true; } if (eventType == HistoryEventType.TASK_ATTEMPT_STARTED) { - suppressedEvents.put(((TaskAttemptStartedEvent)historyEvent).getTaskAttemptID(), event); + suppressedEvents.put(((TaskAttemptStartedEvent) historyEvent).getTaskAttemptID(), event); return false; } else { // TaskAttemptFinishedEvent - TaskAttemptFinishedEvent finishedEvent = (TaskAttemptFinishedEvent)historyEvent; + TaskAttemptFinishedEvent finishedEvent = (TaskAttemptFinishedEvent) historyEvent; if (filters.contains(finishedEvent.getTaskAttemptError())) { suppressedEvents.remove(finishedEvent.getTaskAttemptID()); return false; @@ -233,7 +233,7 @@ private boolean shouldLogTaskAttemptEvents(DAGHistoryEvent event, HistoryLogLeve } private void maybeUpdateDagTaskAttemptFilters(TezDAGID dagId, HistoryLogLevel dagLogLevel, - Configuration dagConf) { + Configuration dagConf) { if (dagLogLevel == HistoryLogLevel.TASK_ATTEMPT) { Set filters = TezUtilsInternal.getEnums( dagConf, @@ -248,7 +248,7 @@ private void maybeUpdateDagTaskAttemptFilters(TezDAGID dagId, HistoryLogLevel da private DAGHistoryEvent getSupressedEvent(HistoryEvent historyEvent) { if (historyEvent.getEventType() == HistoryEventType.TASK_ATTEMPT_FINISHED) { - TaskAttemptFinishedEvent finishedEvent = (TaskAttemptFinishedEvent)historyEvent; + TaskAttemptFinishedEvent finishedEvent = (TaskAttemptFinishedEvent) historyEvent; return suppressedEvents.remove(finishedEvent.getTaskAttemptID()); } return null; @@ -270,5 +270,4 @@ public boolean hasRecoveryFailed() { return false; } } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEventType.java b/tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEventType.java index a536fdfc4f..d712642241 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEventType.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/HistoryEventType.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -87,5 +87,4 @@ public static boolean isDAGSpecificEvent(HistoryEventType historyEventType) { public HistoryLogLevel getHistoryLogLevel() { return historyLogLevel; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/RecoveryConverters.java b/tez-dag/src/main/java/org/apache/tez/dag/history/RecoveryConverters.java index bab713d558..879177e413 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/RecoveryConverters.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/RecoveryConverters.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,5 +23,4 @@ public class RecoveryConverters { - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/SummaryEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/SummaryEvent.java index bdbea59c09..7d40a1aa7a 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/SummaryEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/SummaryEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -35,5 +35,4 @@ public interface SummaryEvent { * @return true if event should be written immediately to the DAG recovery file */ public boolean writeToRecoveryImmediately(); - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AMLaunchedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AMLaunchedEvent.java index 001cbf0104..a15242e1f1 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AMLaunchedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AMLaunchedEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -39,7 +39,7 @@ public AMLaunchedEvent() { } public AMLaunchedEvent(ApplicationAttemptId appAttemptId, - long launchTime, long appSubmitTime, String user) { + long launchTime, long appSubmitTime, String user) { this.applicationAttemptId = appAttemptId; this.launchTime = launchTime; this.appSubmitTime = appSubmitTime; @@ -112,5 +112,4 @@ public long getAppSubmitTime() { public String getUser() { return user; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AMStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AMStartedEvent.java index 87daba68b8..4160828484 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AMStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AMStartedEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -38,7 +38,7 @@ public AMStartedEvent() { } public AMStartedEvent(ApplicationAttemptId appAttemptId, - long startTime, String user) { + long startTime, String user) { this.applicationAttemptId = appAttemptId; this.startTime = startTime; this.user = user; @@ -103,5 +103,4 @@ public long getStartTime() { public String getUser() { return user; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java index 0b812f0184..2e8e7622c5 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/AppLaunchedEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -41,8 +41,8 @@ public AppLaunchedEvent() { } public AppLaunchedEvent(ApplicationId appId, - long launchTime, long appSubmitTime, String user, - Configuration conf, VersionInfo version) { + long launchTime, long appSubmitTime, String user, + Configuration conf, VersionInfo version) { this.applicationId = appId; this.launchTime = launchTime; this.appSubmitTime = appSubmitTime; @@ -106,5 +106,4 @@ public Configuration getConf() { public VersionInfo getVersion() { return version; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerLaunchedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerLaunchedEvent.java index 11528e2f8c..02760b7280 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerLaunchedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerLaunchedEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -39,8 +39,8 @@ public ContainerLaunchedEvent() { } public ContainerLaunchedEvent(ContainerId containerId, - long launchTime, - ApplicationAttemptId applicationAttemptId) { + long launchTime, + ApplicationAttemptId applicationAttemptId) { this.containerId = containerId; this.launchTime = launchTime; this.applicationAttemptId = applicationAttemptId; @@ -107,5 +107,4 @@ public long getLaunchTime() { public ApplicationAttemptId getApplicationAttemptId() { return applicationAttemptId; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerStoppedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerStoppedEvent.java index 528f6293b6..35f6de3449 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerStoppedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/ContainerStoppedEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -38,17 +38,17 @@ public class ContainerStoppedEvent implements HistoryEvent { public ContainerStoppedEvent() { } - + public ContainerStoppedEvent(ContainerId containerId, - long stopTime, - int exitStatus, - ApplicationAttemptId applicationAttemptId) { + long stopTime, + int exitStatus, + ApplicationAttemptId applicationAttemptId) { this.containerId = containerId; this.stopTime = stopTime; this.exitStatus = exitStatus; this.applicationAttemptId = applicationAttemptId; } - + @Override public HistoryEventType getEventType() { return HistoryEventType.CONTAINER_STOPPED; @@ -98,7 +98,7 @@ public void fromProtoStream(CodedInputStream inputStream) throws IOException { @Override public String toString() { return "containerId=" + containerId - + ", stoppedTime=" + stopTime + + ", stoppedTime=" + stopTime + ", exitStatus=" + exitStatus; } @@ -109,7 +109,7 @@ public ContainerId getContainerId() { public long getStoppedTime() { return stopTime; } - + public int getExitStatus() { return exitStatus; } @@ -117,5 +117,4 @@ public int getExitStatus() { public ApplicationAttemptId getApplicationAttemptId() { return applicationAttemptId; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGCommitStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGCommitStartedEvent.java index 241dada2d6..5eb6595808 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGCommitStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGCommitStartedEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -108,5 +108,4 @@ public void fromSummaryProtoStream(SummaryEventProto proto) throws IOException { public boolean writeToRecoveryImmediately() { return false; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java index 6d1d6cd3ff..f95ba15984 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGFinishedEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -60,10 +60,10 @@ public DAGFinishedEvent() { } public DAGFinishedEvent(TezDAGID dagId, long startTime, - long finishTime, DAGState state, - String diagnostics, TezCounters counters, - String user, String dagName, Map dagTaskStats, - ApplicationAttemptId applicationAttemptId, DAGPlan dagPlan) { + long finishTime, DAGState state, + String diagnostics, TezCounters counters, + String user, String dagName, Map dagTaskStats, + ApplicationAttemptId applicationAttemptId, DAGPlan dagPlan) { this.dagID = dagId; this.startTime = startTime; this.finishTime = finishTime; @@ -145,7 +145,7 @@ public String toString() { + ", status=" + state.name() + ", diagnostics=" + diagnostics + ", counters=" + ((tezCounters == null) ? "null" : - (tezCounters.toString() + (tezCounters.toString() .replaceAll("\\n", ", ").replaceAll("\\s+", " "))); } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGInitializedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGInitializedEvent.java index 4df116ad37..af4356d1cf 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGInitializedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGInitializedEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -43,7 +43,7 @@ public DAGInitializedEvent() { } public DAGInitializedEvent(TezDAGID dagID, long initTime, - String user, String dagName, Map vertexNameIDMap) { + String user, String dagName, Map vertexNameIDMap) { this.dagID = dagID; this.initTime = initTime; this.user = user; @@ -118,5 +118,4 @@ public String getDagName() { public Map getVertexNameIDMap() { return vertexNameIDMap; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGKillRequestEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGKillRequestEvent.java index c87f5ce7e7..015494482b 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGKillRequestEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGKillRequestEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -82,7 +82,7 @@ public void fromProtoStream(CodedInputStream inputStream) throws IOException { } fromProto(proto); } - + public void fromProto(RecoveryProtos.DAGKillRequestProto proto) { this.dagID = TezDAGID.fromString(proto.getDagId()); this.killRequestTime = proto.getKillRequestTime(); @@ -93,7 +93,7 @@ public void fromProto(RecoveryProtos.DAGKillRequestProto proto) { public void toSummaryProtoStream(OutputStream outputStream) throws IOException { ProtoUtils.toSummaryEventProto(dagID, killRequestTime, - HistoryEventType.DAG_KILL_REQUEST, isSessionStopped ? new byte[]{1} : new byte[]{0}) + HistoryEventType.DAG_KILL_REQUEST, isSessionStopped ? new byte[]{1} : new byte[]{0}) .writeDelimitedTo(outputStream); } @@ -117,7 +117,7 @@ public boolean writeToRecoveryImmediately() { public TezDAGID getDagID() { return dagID; } - + public long getKillRequestTime() { return killRequestTime; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGRecoveredEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGRecoveredEvent.java index e5f5614819..13805b4f44 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGRecoveredEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGRecoveredEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -42,9 +42,9 @@ public class DAGRecoveredEvent implements HistoryEvent { private String containerLogs; public DAGRecoveredEvent(ApplicationAttemptId applicationAttemptId, - TezDAGID dagId, String dagName, String user, - long recoveredTime, DAGState recoveredState, - String recoveryFailureReason, String containerLogs) { + TezDAGID dagId, String dagName, String user, + long recoveredTime, DAGState recoveredState, + String recoveryFailureReason, String containerLogs) { this.applicationAttemptId = applicationAttemptId; this.dagID = dagId; this.dagName = dagName; @@ -56,7 +56,7 @@ public DAGRecoveredEvent(ApplicationAttemptId applicationAttemptId, } public DAGRecoveredEvent(ApplicationAttemptId applicationAttemptId, - TezDAGID dagId, String dagName, String user, long recoveredTime, String containerLogs) { + TezDAGID dagId, String dagName, String user, long recoveredTime, String containerLogs) { this(applicationAttemptId, dagId, dagName, user, recoveredTime, null, null, containerLogs); } @@ -133,8 +133,7 @@ public String toString() { + (applicationAttemptId != null ? applicationAttemptId.toString() : "null") + ", dagId=" + (dagID != null ? dagID.toString() : "null") + ", recoveredTime=" + recoveredTime - + ", recoveredState=" + (recoveredDagState != null ? recoveredDagState.name() : "null" ) + + ", recoveredState=" + (recoveredDagState != null ? recoveredDagState.name() : "null") + ", recoveryFailureReason=" + recoveryFailureReason; } - -} \ No newline at end of file +} diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGStartedEvent.java index 9e1f6c4a50..41fa9dd473 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGStartedEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -40,7 +40,7 @@ public DAGStartedEvent() { } public DAGStartedEvent(TezDAGID dagID, long startTime, - String user, String dagName) { + String user, String dagName) { this.dagID = dagID; this.startTime = startTime; this.user = user; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGSubmittedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGSubmittedEvent.java index bed0f6f25a..db28cff465 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGSubmittedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/DAGSubmittedEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -42,7 +42,6 @@ import org.apache.tez.dag.recovery.records.RecoveryProtos.SummaryEventProto; import org.apache.tez.dag.utils.ProtoUtils; - public class DAGSubmittedEvent implements HistoryEvent, SummaryEvent, DAGIDAware { private static final Logger LOG = LoggerFactory.getLogger(DAGSubmittedEvent.class); @@ -65,9 +64,9 @@ public DAGSubmittedEvent() { } public DAGSubmittedEvent(TezDAGID dagID, long submitTime, - DAGProtos.DAGPlan dagPlan, ApplicationAttemptId applicationAttemptId, - Map cumulativeAdditionalLocalResources, - String user, Configuration conf, String containerLogs, String queueName) { + DAGProtos.DAGPlan dagPlan, ApplicationAttemptId applicationAttemptId, + Map cumulativeAdditionalLocalResources, + String user, Configuration conf, String containerLogs, String queueName) { this.dagID = dagID; this.dagName = dagPlan.getName(); this.submitTime = submitTime; @@ -96,7 +95,7 @@ public boolean isHistoryEvent() { } public DAGSubmittedProto toProto() { - DAGSubmittedProto.Builder builder =DAGSubmittedProto.newBuilder() + DAGSubmittedProto.Builder builder = DAGSubmittedProto.newBuilder() .setDagId(dagID.toString()) .setApplicationAttemptId(applicationAttemptId.toString()) .setDagPlan(dagPlan) @@ -151,7 +150,7 @@ public String toString() { @Override public void toSummaryProtoStream(OutputStream outputStream) throws IOException { ProtoUtils.toSummaryEventProto(dagID, submitTime, - HistoryEventType.DAG_SUBMITTED, dagName.getBytes(CHARSET_NAME)) + HistoryEventType.DAG_SUBMITTED, dagName.getBytes(CHARSET_NAME)) .writeDelimitedTo(outputStream); } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java index ce2eb729f6..939287d20e 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptFinishedEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -76,23 +76,23 @@ public class TaskAttemptFinishedEvent implements HistoryEvent, TaskAttemptIDAwar private String nodeHttpAddress; public TaskAttemptFinishedEvent(TezTaskAttemptID taId, - String vertexName, - long startTime, - long finishTime, - TaskAttemptState state, - @Nullable TaskFailureType taskFailureType, - TaskAttemptTerminationCause error, - String diagnostics, TezCounters counters, - List dataEvents, - List taGeneratedEvents, - long creationTime, - TezTaskAttemptID creationCausalTA, - long allocationTime, - ContainerId containerId, - NodeId nodeId, - String inProgressLogsUrl, - String completedLogsUrl, - String nodeHttpAddress) { + String vertexName, + long startTime, + long finishTime, + TaskAttemptState state, + @Nullable TaskFailureType taskFailureType, + TaskAttemptTerminationCause error, + String diagnostics, TezCounters counters, + List dataEvents, + List taGeneratedEvents, + long creationTime, + TezTaskAttemptID creationCausalTA, + long allocationTime, + ContainerId containerId, + NodeId nodeId, + String inProgressLogsUrl, + String completedLogsUrl, + String nodeHttpAddress) { this.taskAttemptId = taId; this.vertexName = vertexName; this.creationCausalTA = creationCausalTA; @@ -131,11 +131,11 @@ public boolean isRecoveryEvent() { public boolean isHistoryEvent() { return true; } - + public List getDataEvents() { return dataEvents; } - + public TaskAttemptFinishedProto toProto() throws IOException { TaskAttemptFinishedProto.Builder builder = TaskAttemptFinishedProto.newBuilder(); @@ -203,7 +203,7 @@ public void fromProto(TaskAttemptFinishedProto proto) throws IOException { } if (proto.hasCounters()) { this.tezCounters = DagTypeConverters.convertTezCountersFromProto( - proto.getCounters()); + proto.getCounters()); } if (proto.getDataEventsCount() > 0) { this.dataEvents = Lists.newArrayListWithCapacity(proto.getDataEventsCount()); @@ -320,7 +320,7 @@ public TezCounters getCounters() { public String getDiagnostics() { return diagnostics; } - + public TaskAttemptTerminationCause getTaskAttemptError() { return error; } @@ -340,15 +340,15 @@ public TaskFailureType getTaskFailureType() { public long getStartTime() { return startTime; } - + public long getCreationTime() { return creationTime; } - + public long getAllocationTime() { return allocationTime; } - + public TezTaskAttemptID getCreationCausalTA() { return creationCausalTA; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java index 10ba439e1c..bcd389c648 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskAttemptStartedEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -43,10 +43,10 @@ public class TaskAttemptStartedEvent implements HistoryEvent, TaskAttemptIDAware private String nodeHttpAddress; public TaskAttemptStartedEvent(TezTaskAttemptID taId, - String vertexName, long launchTime, - ContainerId containerId, NodeId nodeId, - String inProgressLogsUrl, String completedLogsUrl, - String nodeHttpAddress) { + String vertexName, long launchTime, + ContainerId containerId, NodeId nodeId, + String inProgressLogsUrl, String completedLogsUrl, + String nodeHttpAddress) { this.taskAttemptId = taId; this.vertexName = vertexName; this.launchTime = launchTime; @@ -122,7 +122,7 @@ public TezTaskAttemptID getTaskAttemptID() { public long getStartTime() { return launchTime; } - + public ContainerId getContainerId() { return containerId; } @@ -142,5 +142,4 @@ public String getCompletedLogsUrl() { public String getNodeHttpAddress() { return nodeHttpAddress; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java index 215968274c..cb431598d9 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskFinishedEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -50,9 +50,9 @@ public class TaskFinishedEvent implements HistoryEvent, TaskIDAware { private int numFailedAttempts; public TaskFinishedEvent(TezTaskID taskID, - String vertexName, long startTime, long finishTime, - TezTaskAttemptID successfulAttemptID, - TaskState state, String diagnostics, TezCounters counters, int failedAttempts) { + String vertexName, long startTime, long finishTime, + TezTaskAttemptID successfulAttemptID, + TaskState state, String diagnostics, TezCounters counters, int failedAttempts) { this.vertexName = vertexName; this.taskID = taskID; this.startTime = startTime; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskStartedEvent.java index 577aef963c..8729a62d5b 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/TaskStartedEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -38,12 +38,12 @@ public class TaskStartedEvent implements HistoryEvent, TaskIDAware { private TaskState state; public TaskStartedEvent(TezTaskID taskId, - String vertexName, long scheduledTime, long startTime) { + String vertexName, long scheduledTime, long startTime) { this(taskId, vertexName, scheduledTime, startTime, TaskState.SCHEDULED); } public TaskStartedEvent(TezTaskID taskId, - String vertexName, long scheduledTime, long startTime, TaskState state) { + String vertexName, long scheduledTime, long startTime, TaskState state) { this.vertexName = vertexName; this.taskID = taskId; this.scheduledTime = scheduledTime; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexCommitStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexCommitStartedEvent.java index ca2cafc834..3bd325f9d2 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexCommitStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexCommitStartedEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -118,6 +118,4 @@ public void fromSummaryProtoStream(SummaryEventProto proto) throws IOException { public boolean writeToRecoveryImmediately() { return false; } - - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java index 2f4fac079f..3b63131a29 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexConfigurationDoneEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -48,15 +48,15 @@ public class VertexConfigurationDoneEvent implements HistoryEvent, VertexIDAware private Map rootInputSpecUpdates; private boolean setParallelismCalledFlag; - public VertexConfigurationDoneEvent() { + public VertexConfigurationDoneEvent() { } public VertexConfigurationDoneEvent(TezVertexID vertexID, - long reconfigureDoneTime, int numTasks, - VertexLocationHint vertexLocationHint, - Map sourceEdgeProperties, - Map rootInputSpecUpdates, - boolean setParallelismCalledFlag) { + long reconfigureDoneTime, int numTasks, + VertexLocationHint vertexLocationHint, + Map sourceEdgeProperties, + Map rootInputSpecUpdates, + boolean setParallelismCalledFlag) { super(); this.vertexID = vertexID; this.reconfigureDoneTime = reconfigureDoneTime; @@ -92,11 +92,11 @@ public VertexConfigurationDoneProto toProto() { if (vertexLocationHint != null) { builder.setVertexLocationHint(DagTypeConverters.convertVertexLocationHintToProto( - this.vertexLocationHint)); + this.vertexLocationHint)); } if (sourceEdgeProperties != null) { for (Entry entry : - sourceEdgeProperties.entrySet()) { + sourceEdgeProperties.entrySet()) { EdgeManagerDescriptorProto.Builder edgeMgrBuilder = EdgeManagerDescriptorProto.newBuilder(); edgeMgrBuilder.setEdgeName(entry.getKey()); @@ -131,7 +131,7 @@ public void fromProto(VertexConfigurationDoneProto proto) { this.sourceEdgeProperties = new HashMap( proto.getEdgeManagerDescriptorsCount()); for (EdgeManagerDescriptorProto edgeManagerProto : - proto.getEdgeManagerDescriptorsList()) { + proto.getEdgeManagerDescriptorsList()) { EdgeProperty edgeProperty = DagTypeConverters.convertFromProto( edgeManagerProto.getEdgeProperty()); @@ -175,9 +175,9 @@ public String toString() { + ", reconfigureDoneTime=" + reconfigureDoneTime + ", numTasks=" + numTasks + ", vertexLocationHint=" + - (vertexLocationHint == null? "null" : vertexLocationHint) + (vertexLocationHint == null ? "null" : vertexLocationHint) + ", edgeManagersCount=" + - (sourceEdgeProperties == null? "null" : sourceEdgeProperties.size()) + (sourceEdgeProperties == null ? "null" : sourceEdgeProperties.size()) + ", rootInputSpecUpdateCount=" + (rootInputSpecUpdates == null ? "null" : rootInputSpecUpdates.size()) + ", setParallelismCalledFlag=" + setParallelismCalledFlag; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java index 24612a6a2e..f18a4986c4 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexFinishedEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -119,7 +119,7 @@ public void fromProto(VertexFinishedProto proto) { this.numTasks = proto.getNumTasks(); this.finishTime = proto.getFinishTime(); this.state = VertexState.values()[proto.getState()]; - if (proto.hasDiagnostics()) { + if (proto.hasDiagnostics()) { this.diagnostics = proto.getDiagnostics(); } } @@ -150,8 +150,8 @@ public String toString() { + ", timeTaken=" + (finishTime - startTime) + ", status=" + state.name() + ", diagnostics=" + diagnostics - + ", counters=" + ( tezCounters == null ? "null" : - tezCounters.toString().replaceAll("\\n", ", ").replaceAll("\\s+", " ")) + + ", counters=" + (tezCounters == null ? "null" : + tezCounters.toString().replaceAll("\\n", ", ").replaceAll("\\s+", " ")) + ", vertexStats=" + (vertexStats == null ? "null" : vertexStats.toString()) + ", vertexTaskStats=" + (vertexTaskStats == null ? "null" : vertexTaskStats.toString()) + ", servicePluginInfo=" @@ -214,7 +214,6 @@ public void toSummaryProtoStream(OutputStream outputStream) throws IOException { .setEventType(getEventType().ordinal()) .setEventPayload(finishStateProto.toByteString()); builder.build().writeDelimitedTo(outputStream); - } @Override @@ -240,5 +239,4 @@ public boolean isFromSummary() { public ServicePluginInfo getServicePluginInfo() { return servicePluginInfo; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitFinishedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitFinishedEvent.java index c9d5aae770..91802a1c9f 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitFinishedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitFinishedEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -47,7 +47,7 @@ public VertexGroupCommitFinishedEvent() { } public VertexGroupCommitFinishedEvent(TezDAGID dagID, - String vertexGroupName, Collection vertexIds, long commitFinishTime) { + String vertexGroupName, Collection vertexIds, long commitFinishTime) { this.dagID = dagID; this.vertexGroupName = vertexGroupName; this.vertexIds = vertexIds; @@ -70,7 +70,7 @@ public boolean isHistoryEvent() { } public VertexGroupCommitFinishedProto toProto() { - Collection vertexIdsStr = Collections2.transform(vertexIds, new Function(){ + Collection vertexIdsStr = Collections2.transform(vertexIds, new Function() { @Override public String apply(TezVertexID vertexId) { return vertexId.toString(); @@ -148,5 +148,4 @@ public TezDAGID getDagID() { public Collection getVertexIds() { return vertexIds; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitStartedEvent.java index cdd11bcc4c..41d01e68f6 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexGroupCommitStartedEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -47,7 +47,7 @@ public VertexGroupCommitStartedEvent() { } public VertexGroupCommitStartedEvent(TezDAGID dagID, - String vertexGroupName, Collection vertexIds, long commitStartTime) { + String vertexGroupName, Collection vertexIds, long commitStartTime) { this.dagID = dagID; this.vertexGroupName = vertexGroupName; this.vertexIds = vertexIds; @@ -70,7 +70,7 @@ public boolean isHistoryEvent() { } public VertexGroupCommitStartedProto toProto() { - Collection vertexIdsStr = Collections2.transform(vertexIds, new Function(){ + Collection vertexIdsStr = Collections2.transform(vertexIds, new Function() { @Override public String apply(TezVertexID vertexId) { return vertexId.toString(); diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexInitializedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexInitializedEvent.java index a019fe3784..dadd454480 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexInitializedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexInitializedEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -59,10 +59,10 @@ public VertexInitializedEvent() { } public VertexInitializedEvent(TezVertexID vertexId, - String vertexName, long initRequestedTime, long initedTime, - int numTasks, String processorName, - Map> additionalInputs, - List initGeneratedEvents, ServicePluginInfo servicePluginInfo) { + String vertexName, long initRequestedTime, long initedTime, + int numTasks, String processorName, + Map> additionalInputs, + List initGeneratedEvents, ServicePluginInfo servicePluginInfo) { this.vertexName = vertexName; this.vertexID = vertexId; this.initRequestedTime = initRequestedTime; @@ -92,9 +92,9 @@ public boolean isHistoryEvent() { public RecoveryProtos.VertexInitializedProto toProto() throws IOException { VertexInitializedProto.Builder builder = VertexInitializedProto.newBuilder(); if (additionalInputs != null - && !additionalInputs.isEmpty()) { + && !additionalInputs.isEmpty()) { for (RootInputLeafOutput input : - additionalInputs.values()) { + additionalInputs.values()) { RootInputLeafOutputProto.Builder inputBuilder = RootInputLeafOutputProto.newBuilder(); inputBuilder.setName(input.getName()); @@ -198,8 +198,8 @@ public int getNumTasks() { return numTasks; } - public Map> - getAdditionalInputs() { + public Map> + getAdditionalInputs() { return additionalInputs; } @@ -218,5 +218,4 @@ public List getInitGeneratedEvents() { public ServicePluginInfo getServicePluginInfo() { return servicePluginInfo; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexStartedEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexStartedEvent.java index cdd73b4483..77aa6b4478 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexStartedEvent.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/events/VertexStartedEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -39,7 +39,7 @@ public VertexStartedEvent() { } public VertexStartedEvent(TezVertexID vertexId, - long startRequestedTime, long startTime) { + long startRequestedTime, long startTime) { this.vertexID = vertexId; this.startRequestedTime = startRequestedTime; this.startTime = startTime; diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/logging/HistoryLoggingService.java b/tez-dag/src/main/java/org/apache/tez/dag/history/logging/HistoryLoggingService.java index 44efad7228..407bce3bc5 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/logging/HistoryLoggingService.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/logging/HistoryLoggingService.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -39,5 +39,4 @@ public HistoryLoggingService(String name) { * @param event History event to be logged */ public abstract void handle(DAGHistoryEvent event); - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/DevNullHistoryLoggingService.java b/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/DevNullHistoryLoggingService.java index 7e8a335c05..f6d6a95f2b 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/DevNullHistoryLoggingService.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/DevNullHistoryLoggingService.java @@ -35,5 +35,4 @@ public void handle(DAGHistoryEvent event) { // Sending all events to /dev/null // Done. } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java b/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java index 8f4cd1fa6a..620f2e7b06 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -95,7 +95,7 @@ public static JSONObject convertToJson(HistoryEvent historyEvent) throws JSONExc break; case VERTEX_FINISHED: jsonObject = convertVertexFinishedEvent((VertexFinishedEvent) historyEvent); - break; + break; case TASK_STARTED: jsonObject = convertTaskStartedEvent((TaskStartedEvent) historyEvent); break; @@ -186,7 +186,7 @@ private static JSONObject convertAMLaunchedEvent(AMLaunchedEvent event) throws J jsonObject.put(ATSConstants.ENTITY, "tez_" + event.getApplicationAttemptId().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, - EntityTypes.TEZ_APPLICATION_ATTEMPT.name()); + EntityTypes.TEZ_APPLICATION_ATTEMPT.name()); // Related Entities JSONArray relatedEntities = new JSONArray(); @@ -199,7 +199,7 @@ private static JSONObject convertAMLaunchedEvent(AMLaunchedEvent event) throws J appAttemptEntity.put(ATSConstants.ENTITY, event.getApplicationAttemptId().toString()); appAttemptEntity.put(ATSConstants.ENTITY_TYPE, - ATSConstants.APPLICATION_ATTEMPT_ID); + ATSConstants.APPLICATION_ATTEMPT_ID); relatedEntities.put(appEntity); relatedEntities.put(appAttemptEntity); jsonObject.put(ATSConstants.RELATED_ENTITIES, relatedEntities); @@ -218,7 +218,7 @@ private static JSONObject convertAMLaunchedEvent(AMLaunchedEvent event) throws J JSONObject otherInfo = new JSONObject(); otherInfo.put(ATSConstants.APP_SUBMIT_TIME, event.getAppSubmitTime()); jsonObject.put(ATSConstants.OTHER_INFO, otherInfo); - + return jsonObject; } @@ -255,7 +255,8 @@ private static JSONObject convertAMStartedEvent(AMStartedEvent event) throws JSO events.put(startEvent); jsonObject.put(ATSConstants.EVENTS, events); - return jsonObject; } + return jsonObject; + } private static JSONObject convertContainerLaunchedEvent(ContainerLaunchedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); @@ -336,7 +337,8 @@ private static JSONObject convertContainerStoppedEvent(ContainerStoppedEvent eve otherInfo.put(ATSConstants.EXIT_STATUS, event.getExitStatus()); jsonObject.put(ATSConstants.OTHER_INFO, otherInfo); - return jsonObject; } + return jsonObject; + } private static JSONObject convertDAGFinishedEvent(DAGFinishedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); @@ -371,7 +373,7 @@ private static JSONObject convertDAGFinishedEvent(DAGFinishedEvent event) throws final Map dagTaskStats = event.getDagTaskStats(); if (dagTaskStats != null) { - for(Entry entry : dagTaskStats.entrySet()) { + for (Entry entry : dagTaskStats.entrySet()) { otherInfo.put(entry.getKey(), entry.getValue().intValue()); } } @@ -561,7 +563,7 @@ private static JSONObject convertTaskAttemptFinishedEvent(TaskAttemptFinishedEve otherInfo.put(ATSConstants.COUNTERS, DAGUtils.convertCountersToJSON(event.getCounters())); if (event.getDataEvents() != null && !event.getDataEvents().isEmpty()) { - otherInfo.put(ATSConstants.LAST_DATA_EVENTS, + otherInfo.put(ATSConstants.LAST_DATA_EVENTS, DAGUtils.convertDataEventDependencyInfoToJSON(event.getDataEvents())); } if (event.getNodeId() != null) { @@ -718,7 +720,7 @@ private static JSONObject convertVertexFinishedEvent(VertexFinishedEvent event) final Map vertexTaskStats = event.getVertexTaskStats(); if (vertexTaskStats != null) { - for(Entry entry : vertexTaskStats.entrySet()) { + for (Entry entry : vertexTaskStats.entrySet()) { otherInfo.put(entry.getKey(), entry.getValue().intValue()); } } @@ -838,5 +840,4 @@ private static JSONObject convertVertexStartedEvent(VertexStartedEvent event) return jsonObject; } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/SimpleHistoryLoggingService.java b/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/SimpleHistoryLoggingService.java index 418bc3c5ea..8d47876e1a 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/SimpleHistoryLoggingService.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/SimpleHistoryLoggingService.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -171,6 +171,5 @@ private synchronized void handleEvent(DAGHistoryEvent event) { "consecutive max errors reached, maxErrors=" + maxErrors); } } - } } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/recovery/RecoveryService.java b/tez-dag/src/main/java/org/apache/tez/dag/history/recovery/RecoveryService.java index 5b5a9c7261..ea3b6eb7b6 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/recovery/RecoveryService.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/recovery/RecoveryService.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -148,10 +148,10 @@ public void serviceInit(Configuration conf) throws Exception { TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED_DEFAULT); LOG.info("RecoveryService initialized with " - + "recoveryPath=" + recoveryPath - + ", bufferSize(bytes)=" + bufferSize - + ", flushInterval(s)=" + flushInterval - + ", maxUnflushedEvents=" + maxUnflushedEvents); + + "recoveryPath=" + recoveryPath + + ", bufferSize(bytes)=" + bufferSize + + ", flushInterval(s)=" + flushInterval + + ", maxUnflushedEvents=" + maxUnflushedEvents); } @Override @@ -307,7 +307,7 @@ public void handle(DAGHistoryEvent event) throws IOException { String dagName = dagSubmittedEvent.getDAGName(); if (dagName != null && dagName.startsWith( - TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX)) { + TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX)) { // Skip recording pre-warm DAG events skippedDAGs.add(dagId); return; @@ -403,9 +403,9 @@ private void createFatalErrorFlagDir() throws IOException { } protected void handleSummaryEvent(TezDAGID dagID, - HistoryEventType eventType, - SummaryEvent summaryEvent) throws IOException { - LOG.debug("Handling summary event, dagID={}, eventType={}", dagID, eventType); + HistoryEventType eventType, + SummaryEvent summaryEvent) throws IOException { + LOG.debug("Handling summary event, dagID={}, eventType={}", dagID, eventType); if (summaryStream == null) { Path summaryPath = TezCommonUtils.getSummaryRecoveryPath(recoveryPath); @@ -466,9 +466,8 @@ protected void handleRecoveryEvent(DAGHistoryEvent event) throws IOException { outputStreamMap.put(dagID, recoveryStream); } - LOG.debug("Writing recovery event to output stream, dagId={}, eventType={}", - dagID, eventType); + dagID, eventType); ++unflushedEventsCount; recoveryStream.codedOutputStream.writeFixed32NoTag(event.getHistoryEvent().getEventType().ordinal()); event.getHistoryEvent().toProtoStream(recoveryStream.codedOutputStream); @@ -481,14 +480,14 @@ protected void handleRecoveryEvent(DAGHistoryEvent event) throws IOException { private void maybeFlush(RecoveryStream recoveryStream) throws IOException { long currentTime = appContext.getClock().getTime(); boolean doFlush = false; - if (maxUnflushedEvents >=0 + if (maxUnflushedEvents >= 0 && unflushedEventsCount >= maxUnflushedEvents) { - LOG.debug("Max unflushed events count reached. Flushing recovery data, " - + "unflushedEventsCount={}, maxUnflushedEvents={}", unflushedEventsCount, - maxUnflushedEvents); + LOG.debug("Max unflushed events count reached. Flushing recovery data, " + + "unflushedEventsCount={}, maxUnflushedEvents={}", unflushedEventsCount, + maxUnflushedEvents); doFlush = true; } else if (flushInterval >= 0 - && ((currentTime - lastFlushTime) >= (flushInterval*1000))) { + && ((currentTime - lastFlushTime) >= (flushInterval * 1000))) { LOG.debug("Flush interval time period elapsed. Flushing recovery data" + ", lastTimeSinceFLush=" + lastFlushTime + ", timeSinceLastFlush=" + (currentTime - lastFlushTime)); @@ -501,7 +500,7 @@ private void maybeFlush(RecoveryStream recoveryStream) throws IOException { } private void doFlush(RecoveryStream recoveryStream, - long currentTime) throws IOException { + long currentTime) throws IOException { recoveryStream.flush(); if (LOG.isDebugEnabled()) { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java index f59e9ace7c..3ca7fe8f97 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -99,8 +99,6 @@ public class DAGUtils { public static final String VERTEX_GROUP_EDGE_MERGED_INPUTS_KEY = "edgeMergedInputs"; public static final String VERTEX_GROUP_DESTINATION_VERTEX_NAME_KEY = "destinationVertexName"; - - public static JSONObject generateSimpleJSONPlan(DAGPlan dagPlan) throws JSONException { JSONObject dagJson; try { @@ -110,11 +108,11 @@ public static JSONObject generateSimpleJSONPlan(DAGPlan dagPlan) throws JSONExce } return dagJson; } - + public static JSONObject convertDataEventDependencyInfoToJSON(List info) { return new JSONObject(convertDataEventDependecyInfoToATS(info)); } - + public static Map convertDataEventDependecyInfoToATS(List info) { ArrayList infoList = new ArrayList(); for (DataEventDependencyInfo event : info) { @@ -127,7 +125,7 @@ public static Map convertDataEventDependecyInfoToATS(List object = new LinkedHashMap(); + Map object = new LinkedHashMap(); putInto(object, ATSConstants.LAST_DATA_EVENTS, infoList); return object; } @@ -138,46 +136,45 @@ public static JSONObject convertCountersToJSON(TezCounters counters) return jsonObject; } - public static Map convertCountersToATSMap(TezCounters counters) { + public static Map convertCountersToATSMap(TezCounters counters) { Map object = new LinkedHashMap(); if (counters == null) { - return object; - } + return object; + } ArrayList counterGroupsList = new ArrayList(); for (CounterGroup group : counters) { - ArrayList counterList = new ArrayList(); - for (TezCounter counter : group) { - if (counter.getValue() != 0) { - Map counterMap = new LinkedHashMap(); - counterMap.put(ATSConstants.COUNTER_NAME, counter.getName()); - if (!counter.getDisplayName().equals(counter.getName())) { - counterMap.put(ATSConstants.COUNTER_DISPLAY_NAME, - counter.getDisplayName()); - } - counterMap.put(ATSConstants.COUNTER_VALUE, counter.getValue()); - if (counter instanceof AggregateTezCounter) { - counterMap.put(ATSConstants.COUNTER_INSTANCE_COUNT, - ((AggregateTezCounter)counter).getCount()); - counterMap.put(ATSConstants.COUNTER_MAX_VALUE, - ((AggregateTezCounter)counter).getMax()); - counterMap.put(ATSConstants.COUNTER_MIN_VALUE, - ((AggregateTezCounter)counter).getMin()); - - } - counterList.add(counterMap); + ArrayList counterList = new ArrayList(); + for (TezCounter counter : group) { + if (counter.getValue() != 0) { + Map counterMap = new LinkedHashMap(); + counterMap.put(ATSConstants.COUNTER_NAME, counter.getName()); + if (!counter.getDisplayName().equals(counter.getName())) { + counterMap.put(ATSConstants.COUNTER_DISPLAY_NAME, + counter.getDisplayName()); } - } - if (!counterList.isEmpty()) { - Map counterGroupMap = new LinkedHashMap(); - counterGroupMap.put(ATSConstants.COUNTER_GROUP_NAME, group.getName()); - if (!group.getDisplayName().equals(group.getName())) { - counterGroupMap.put(ATSConstants.COUNTER_GROUP_DISPLAY_NAME, - group.getDisplayName()); + counterMap.put(ATSConstants.COUNTER_VALUE, counter.getValue()); + if (counter instanceof AggregateTezCounter) { + counterMap.put(ATSConstants.COUNTER_INSTANCE_COUNT, + ((AggregateTezCounter) counter).getCount()); + counterMap.put(ATSConstants.COUNTER_MAX_VALUE, + ((AggregateTezCounter) counter).getMax()); + counterMap.put(ATSConstants.COUNTER_MIN_VALUE, + ((AggregateTezCounter) counter).getMin()); } - counterGroupMap.put(ATSConstants.COUNTERS, counterList); - counterGroupsList.add(counterGroupMap); + counterList.add(counterMap); + } + } + if (!counterList.isEmpty()) { + Map counterGroupMap = new LinkedHashMap(); + counterGroupMap.put(ATSConstants.COUNTER_GROUP_NAME, group.getName()); + if (!group.getDisplayName().equals(group.getName())) { + counterGroupMap.put(ATSConstants.COUNTER_GROUP_DISPLAY_NAME, + group.getDisplayName()); } + counterGroupMap.put(ATSConstants.COUNTERS, counterList); + counterGroupsList.add(counterGroupMap); } + } putInto(object, ATSConstants.COUNTER_GROUPS, counterGroupsList); return object; } @@ -199,7 +196,7 @@ static Map createDagInfoMap(DAGPlan dagPlan) { } public static Map convertDAGPlanToATSMap(final DAGPlan - dagPlan) throws IOException { + dagPlan) throws IOException { final Inflater inflater = TezCommonUtils.newInflater(); try { return convertDAGPlanToATSMap(dagPlan, inflater); @@ -215,10 +212,10 @@ public static Map convertDAGPlanToATSMap(final DAGPlan * @return ATS MAP */ private static Map convertDAGPlanToATSMap(DAGPlan dagPlan, - final Inflater inflater) { + final Inflater inflater) { final String VERSION_KEY = "version"; final int version = 2; - Map dagMap = new LinkedHashMap(); + Map dagMap = new LinkedHashMap(); dagMap.put(DAG_NAME_KEY, dagPlan.getName()); if (dagPlan.hasDagInfo()) { dagMap.put(DAG_INFO_KEY, dagPlan.getDagInfo()); @@ -229,7 +226,7 @@ private static Map convertDAGPlanToATSMap(DAGPlan dagPlan, dagMap.put(VERSION_KEY, version); ArrayList verticesList = new ArrayList(); for (DAGProtos.VertexPlan vertexPlan : dagPlan.getVertexList()) { - Map vertexMap = new LinkedHashMap(); + Map vertexMap = new LinkedHashMap(); vertexMap.put(VERTEX_NAME_KEY, vertexPlan.getName()); if (vertexPlan.hasProcessorDescriptor()) { vertexMap.put(PROCESSOR_CLASS_KEY, @@ -252,7 +249,7 @@ private static Map convertDAGPlanToATSMap(DAGPlan dagPlan, ArrayList inputsList = new ArrayList(); for (DAGProtos.RootInputLeafOutputProto input : vertexPlan.getInputsList()) { - Map inputMap = new LinkedHashMap(); + Map inputMap = new LinkedHashMap(); inputMap.put(NAME_KEY, input.getName()); inputMap.put(CLASS_KEY, input.getIODescriptor().getClassName()); if (input.hasControllerDescriptor()) { @@ -270,7 +267,7 @@ private static Map convertDAGPlanToATSMap(DAGPlan dagPlan, ArrayList outputsList = new ArrayList(); for (DAGProtos.RootInputLeafOutputProto output : vertexPlan.getOutputsList()) { - Map outputMap = new LinkedHashMap(); + Map outputMap = new LinkedHashMap(); outputMap.put(NAME_KEY, output.getName()); outputMap.put(CLASS_KEY, output.getIODescriptor().getClassName()); if (output.hasControllerDescriptor()) { @@ -296,7 +293,7 @@ private static Map convertDAGPlanToATSMap(DAGPlan dagPlan, ArrayList edgesList = new ArrayList(); for (DAGProtos.EdgePlan edgePlan : dagPlan.getEdgeList()) { - Map edgeMap = new LinkedHashMap(); + Map edgeMap = new LinkedHashMap(); edgeMap.put(EDGE_ID_KEY, edgePlan.getId()); edgeMap.put(INPUT_VERTEX_NAME_KEY, edgePlan.getInputVertexName()); edgeMap.put(OUTPUT_VERTEX_NAME_KEY, edgePlan.getOutputVertexName()); @@ -325,9 +322,9 @@ private static Map convertDAGPlanToATSMap(DAGPlan dagPlan, ArrayList vertexGroupsList = new ArrayList(); for (DAGProtos.PlanVertexGroupInfo vertexGroupInfo : dagPlan.getVertexGroupsList()) { - Map groupMap = new LinkedHashMap(); + Map groupMap = new LinkedHashMap(); groupMap.put(VERTEX_GROUP_NAME_KEY, vertexGroupInfo.getGroupName()); - if (vertexGroupInfo.getGroupMembersCount() > 0 ) { + if (vertexGroupInfo.getGroupMembersCount() > 0) { groupMap.put(VERTEX_GROUP_MEMBERS_KEY, vertexGroupInfo.getGroupMembersList()); } if (vertexGroupInfo.getOutputsCount() > 0) { @@ -338,11 +335,11 @@ private static Map convertDAGPlanToATSMap(DAGPlan dagPlan, ArrayList edgeMergedInputs = new ArrayList(); for (PlanGroupInputEdgeInfo edgeMergedInputInfo : vertexGroupInfo.getEdgeMergedInputsList()) { - Map edgeMergedInput = new LinkedHashMap(); + Map edgeMergedInput = new LinkedHashMap(); edgeMergedInput.put(VERTEX_GROUP_DESTINATION_VERTEX_NAME_KEY, edgeMergedInputInfo.getDestVertexName()); if (edgeMergedInputInfo.hasMergedInput() - && edgeMergedInputInfo.getMergedInput().hasClassName()) { + && edgeMergedInputInfo.getMergedInput().hasClassName()) { edgeMergedInput.put(PROCESSOR_CLASS_KEY, edgeMergedInputInfo.getMergedInput().getClassName()); if (edgeMergedInputInfo.getMergedInput().hasHistoryText()) { @@ -363,7 +360,7 @@ private static Map convertDAGPlanToATSMap(DAGPlan dagPlan, } private static void putInto(Map map, String key, - ArrayList list) { + ArrayList list) { if (list.isEmpty()) { return; } @@ -385,9 +382,9 @@ public static JSONObject convertVertexStatsToJSON(VertexStats vertexStats) return jsonObject; } - public static Map convertVertexStatsToATSMap( + public static Map convertVertexStatsToATSMap( VertexStats vertexStats) { - Map vertexStatsMap = new LinkedHashMap(); + Map vertexStatsMap = new LinkedHashMap(); if (vertexStats == null) { return vertexStatsMap; } @@ -441,7 +438,7 @@ public static JSONObject convertServicePluginToJSON( return jsonObject; } - public static Map convertServicePluginToATSMap( + public static Map convertServicePluginToATSMap( ServicePluginInfo servicePluginInfo) { Map servicePluginMap = new LinkedHashMap(); if (servicePluginInfo == null) { @@ -462,11 +459,10 @@ public static Map convertServicePluginToATSMap( return servicePluginMap; } - - public static Map convertEdgeProperty( + public static Map convertEdgeProperty( EdgeProperty edge) { Map jsonDescriptor = new HashMap(); - + jsonDescriptor.put(DATA_MOVEMENT_TYPE_KEY, edge.getDataMovementType().name()); jsonDescriptor.put(DATA_SOURCE_TYPE_KEY, edge.getDataSourceType().name()); @@ -492,8 +488,8 @@ public static Map convertEdgeProperty( } return jsonDescriptor; } - - public static Map convertEdgeManagerPluginDescriptor( + + public static Map convertEdgeManagerPluginDescriptor( EdgeManagerPluginDescriptor descriptor) { Map jsonDescriptor = new HashMap(); jsonDescriptor.put(EDGE_MANAGER_CLASS_KEY, descriptor.getClassName()); @@ -521,6 +517,4 @@ public static Map convertTezVersionToATSMap(VersionInfo versionI atsInfo.put(ATSConstants.REVISION, versionInfo.getRevision()); return atsInfo; } - - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/TezEventUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/TezEventUtils.java index cc89b9fb4d..6742c4472a 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/TezEventUtils.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/TezEventUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,6 +18,7 @@ package org.apache.tez.dag.history.utils; import java.io.IOException; + import org.apache.tez.common.ProtoConverters; import org.apache.tez.dag.records.TezTaskAttemptID; import org.apache.tez.dag.recovery.records.RecoveryProtos; @@ -50,7 +51,7 @@ public static TezEventProto toProto(TezEvent event) throws IOException { .convertRootInputInitializerEventToProto((InputInitializerEvent) event.getEvent())); } else if (event.getEventType().equals(EventType.VERTEX_MANAGER_EVENT)) { evtBuilder.setVmEvent(ProtoConverters - .convertVertexManagerEventToProto((VertexManagerEvent)event.getEvent())); + .convertVertexManagerEventToProto((VertexManagerEvent) event.getEvent())); } else if (event.getEventType().equals(EventType.ROOT_INPUT_DATA_INFORMATION_EVENT)) { evtBuilder.setRootInputDataInformationEvent( ProtoConverters.convertRootInputDataInformationEventToProto( @@ -102,16 +103,16 @@ public static TezEvent fromProto(TezEventProto eventProto) throws IOException { tezEvent.setDestinationInfo(destinationInfo); return tezEvent; } - + public static RecoveryProtos.EventMetaDataProto convertEventMetaDataToProto( EventMetaData eventMetaData) { RecoveryProtos.EventMetaDataProto.Builder builder = RecoveryProtos.EventMetaDataProto.newBuilder() - .setProducerConsumerType(eventMetaData.getEventGenerator().ordinal()) - .setEdgeVertexName(eventMetaData.getEdgeVertexName()) - .setTaskVertexName(eventMetaData.getTaskVertexName()); + .setProducerConsumerType(eventMetaData.getEventGenerator().ordinal()) + .setEdgeVertexName(eventMetaData.getEdgeVertexName()) + .setTaskVertexName(eventMetaData.getTaskVertexName()); if (eventMetaData.getTaskAttemptID() != null) { - builder.setTaskAttemptId(eventMetaData.getTaskAttemptID().toString()); + builder.setTaskAttemptId(eventMetaData.getTaskAttemptID().toString()); } return builder.build(); } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/Graph.java b/tez-dag/src/main/java/org/apache/tez/dag/utils/Graph.java index eb2bd41889..c20ffdf2a8 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/utils/Graph.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/Graph.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -169,7 +169,7 @@ public void addSubGraph(Graph graph) { private static String wrapSafeString(String label) { if (label.indexOf(',') >= 0) { - if (label.length()>14) { + if (label.length() > 14) { label = label.replaceAll(",", ",\n"); } } @@ -194,7 +194,7 @@ public String generateGraphViz(String indent) { sb.append(System.getProperty("line.separator")); } for (Graph g : subgraphs) { - String ginfo = g.generateGraphViz(indent+" "); + String ginfo = g.generateGraphViz(indent + " "); sb.append(ginfo); sb.append(System.getProperty("line.separator")); } @@ -237,7 +237,7 @@ public String generateGraphViz() { } public void save(String filePath) throws IOException { - try (FileOutputStream fout = new FileOutputStream(filePath)){ + try (FileOutputStream fout = new FileOutputStream(filePath)) { fout.write(generateGraphViz().getBytes("UTF-8")); fout.close(); } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/ProtoUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/utils/ProtoUtils.java index 56e46a0556..c4bcacb819 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/utils/ProtoUtils.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/ProtoUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -30,13 +30,12 @@ public static RecoveryProtos.SummaryEventProto toSummaryEventProto( TezDAGID dagID, long timestamp, HistoryEventType historyEventType, byte[] payload) { RecoveryProtos.SummaryEventProto.Builder builder = RecoveryProtos.SummaryEventProto.newBuilder() - .setDagId(dagID.toString()) - .setTimestamp(timestamp) - .setEventType(historyEventType.ordinal()); - if (payload != null){ + .setDagId(dagID.toString()) + .setTimestamp(timestamp) + .setEventType(historyEventType.ordinal()); + if (payload != null) { builder.setEventPayload(ByteString.copyFrom(payload)); } return builder.build(); } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/Simple2LevelVersionComparator.java b/tez-dag/src/main/java/org/apache/tez/dag/utils/Simple2LevelVersionComparator.java index b54187c875..cd6a363843 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/utils/Simple2LevelVersionComparator.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/Simple2LevelVersionComparator.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -66,7 +66,6 @@ public Version(String versionString) { } } } - } public int compare(String versionStr1, String versionStr2) { diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/TaskSpecificLaunchCmdOption.java b/tez-dag/src/main/java/org/apache/tez/dag/utils/TaskSpecificLaunchCmdOption.java index 7d93481581..8abd014e04 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/utils/TaskSpecificLaunchCmdOption.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/TaskSpecificLaunchCmdOption.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -42,7 +42,7 @@ public class TaskSpecificLaunchCmdOption { //To check any characters apart from "a-zA-Z_0-9 : ; , [] - space" anywhere in input. final static Pattern INVALID_TASK_NAME_REGEX = Pattern - .compile("[^(\\w\\s;:,\\[\\]\\-)]"); + .compile("[^(\\w\\s;:,\\[\\]\\-)]"); /** * Regex to validate the task ranges. Vertex name can only have [a-zA-Z_0-9], -, and @@ -50,7 +50,7 @@ public class TaskSpecificLaunchCmdOption { * as task id separator. */ final static Pattern TASKS_REGEX = Pattern - .compile("([\\w\\s\\-]+)\\[([\\d:,\\s]*)\\];?"); + .compile("([\\w\\s\\-]+)\\[([\\d:,\\s]*)\\];?"); //Range regex where ':' should always be prepended and appended with digit. final static Pattern RANGE_REGEX = Pattern.compile("(\\d+):(\\d+)"); @@ -84,7 +84,7 @@ public TaskSpecificLaunchCmdOption(Configuration conf) { * @return tsLaunchCmdOpts */ public String getTaskSpecificOption(String launchCmdOpts, String vertexName, - int taskIdx) { + int taskIdx) { if (this.tsLaunchCmdOpts != null) { launchCmdOpts = (launchCmdOpts == null) ? "" : launchCmdOpts; vertexName = vertexName.replaceAll(" ", ""); @@ -203,7 +203,7 @@ private BitSet parseTasks(String tasksInVertex) { * allowed. */ if (task.endsWith(":") || task.startsWith(":")) { - //invalid range. e.g :20, 6: are not supported. + //invalid range. e.g :20, 6: are not supported. LOG.warn("Partial range is considered as an invalid option"); return null; } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java index b7e6f724f7..0202c4f536 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezBuilderUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,19 +36,18 @@ public static TezVertexID newVertexID(TezDAGID dagId, int vertexId) { public static TezTaskAttemptID newTaskAttemptId(TezTaskID taskId, int id) { return TezTaskAttemptID.getInstance(taskId, id); } - + public static DAGReport newDAGReport() { return null; } - public static AMInfo newAMInfo(ApplicationAttemptId appAttemptID, - long startTime, ContainerId containerID, String nmHost, - int nmPort, int nmHttpPort) { + public static AMInfo newAMInfo(ApplicationAttemptId appAttemptID, + long startTime, ContainerId containerID, String nmHost, + int nmPort, int nmHttpPort) { return null; } public static TezTaskID newTaskId(TezDAGID dagId, int vertexId, int taskId) { return TezTaskID.getInstance(newVertexID(dagId, vertexId), taskId); } - } diff --git a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezRuntimeChildJVM.java b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezRuntimeChildJVM.java index 9458193dc1..888fbe8e43 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/utils/TezRuntimeChildJVM.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/utils/TezRuntimeChildJVM.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.utils; @@ -30,22 +30,22 @@ public class TezRuntimeChildJVM { - // FIXME + // FIXME public static enum LogName { /** Log on the stdout of the task. */ - STDOUT ("stdout"), + STDOUT("stdout"), /** Log on the stderr of the task. */ - STDERR ("stderr"), + STDERR("stderr"), /** Log on the map-reduce system logs of the task. */ - SYSLOG ("syslog"), + SYSLOG("syslog"), /** The java profiler information. */ - PROFILE ("profile.out"), + PROFILE("profile.out"), /** Log the debug script's stdout */ - DEBUGOUT ("debugout"); + DEBUGOUT("debugout"); private String prefix; diff --git a/tez-dag/src/main/java/org/apache/tez/serviceplugins/api/TaskCommunicator.java b/tez-dag/src/main/java/org/apache/tez/serviceplugins/api/TaskCommunicator.java index be6ad68d55..b2ea9f307f 100644 --- a/tez-dag/src/main/java/org/apache/tez/serviceplugins/api/TaskCommunicator.java +++ b/tez-dag/src/main/java/org/apache/tez/serviceplugins/api/TaskCommunicator.java @@ -37,10 +37,9 @@ *

    * The plugin is initialized with an instance of {@link TaskCommunicatorContext} - which provides * a mechanism to notify the system about allocation decisions and resources to the Tez framework. - * + *

    * If setting up a heartbeat between the task and the AM, the framework is responsible for error checking * of this heartbeat mechanism, handling lost or duplicate responses. - * */ public abstract class TaskCommunicator implements ServicePluginLifecycle { @@ -99,7 +98,6 @@ public void start() throws Exception { public void shutdown() throws Exception { } - /** * Register a new container. * @@ -107,7 +105,7 @@ public void shutdown() throws Exception { * @param hostname the hostname on which the container runs * @param port the port for the service which is running the container * @throws ServicePluginException when the service runs into a fatal error which it cannot handle. - * This will cause the app to shutdown. + * This will cause the app to shutdown. */ public abstract void registerRunningContainer(ContainerId containerId, String hostname, int port) throws ServicePluginException; @@ -120,7 +118,7 @@ public abstract void registerRunningContainer(ContainerId containerId, String ho * @param endReason the end reason for the container completing * @param diagnostics diagnostics associated with the container end * @throws ServicePluginException when the service runs into a fatal error which it cannot handle. - * This will cause the app to shutdown. + * This will cause the app to shutdown. */ public abstract void registerContainerEnd(ContainerId containerId, ContainerEndReason endReason, @Nullable String diagnostics) throws @@ -139,7 +137,7 @@ public abstract void registerContainerEnd(ContainerId containerId, ContainerEndR * associated with this container * @param priority the priority of the task being executed * @throws ServicePluginException when the service runs into a fatal error which it cannot handle. - * This will cause the app to shutdown. + * This will cause the app to shutdown. */ public abstract void registerRunningTaskAttempt(ContainerId containerId, TaskSpec taskSpec, Map additionalResources, @@ -155,7 +153,7 @@ public abstract void registerRunningTaskAttempt(ContainerId containerId, TaskSpe * @param endReason the endReason for the task attempt. * @param diagnostics diagnostics associated with the task end * @throws ServicePluginException when the service runs into a fatal error which it cannot handle. - * This will cause the app to shutdown. + * This will cause the app to shutdown. */ public abstract void unregisterRunningTaskAttempt(TezTaskAttemptID taskAttemptID, TaskAttemptEndReason endReason, @@ -167,7 +165,7 @@ public abstract void unregisterRunningTaskAttempt(TezTaskAttemptID taskAttemptID * * @return the address * @throws ServicePluginException when the service runs into a fatal error which it cannot handle. - * This will cause the app to shutdown. + * This will cause the app to shutdown. */ public abstract InetSocketAddress getAddress() throws ServicePluginException; @@ -189,7 +187,7 @@ public abstract void unregisterRunningTaskAttempt(TezTaskAttemptID taskAttemptID * Additional information may be available for specific events, Look at the * type hierarchy for {@link org.apache.tez.dag.api.event.VertexStateUpdate} * @throws ServicePluginException when the service runs into a fatal error which it cannot handle. - * This will cause the app to shutdown. + * This will cause the app to shutdown. */ public abstract void onVertexStateUpdated(VertexStateUpdate stateUpdate) throws ServicePluginException; @@ -202,7 +200,7 @@ public abstract void unregisterRunningTaskAttempt(TezTaskAttemptID taskAttemptID * * @param dagIdentifier the unique numerical identifier for the DAG in the specified execution context. * @throws ServicePluginException when the service runs into a fatal error which it cannot handle. - * This will cause the app to shutdown. + * This will cause the app to shutdown. */ public abstract void dagComplete(int dagIdentifier) throws ServicePluginException; @@ -213,13 +211,14 @@ public abstract void unregisterRunningTaskAttempt(TezTaskAttemptID taskAttemptID * * @return meta info for the task communicator * @throws ServicePluginException when the service runs into a fatal error which it cannot handle. - * This will cause the app to shutdown. + * This will cause the app to shutdown. */ public abstract Object getMetaInfo() throws ServicePluginException; /** * Return a URL that can be used as a link to the logs for a running attempt. - * @param attemptID Attempt ID for which the log link should be provided + * + * @param attemptID Attempt ID for which the log link should be provided * @param containerNodeId Node Id on which the attempt is meant to have run * @return URL to logs for the attempt */ @@ -229,7 +228,8 @@ public String getInProgressLogsUrl(TezTaskAttemptID attemptID, NodeId containerN /** * Return a URL that can be used as a link to the logs for a completed attempt. - * @param attemptID Attempt ID for which the log link should be provided + * + * @param attemptID Attempt ID for which the log link should be provided * @param containerNodeId Node Id on which the attempt is meant to have run * @return URL to logs for the attempt */ @@ -241,6 +241,7 @@ public String getCompletedLogsUrl(TezTaskAttemptID attemptID, NodeId containerNo * Return the amount of memory used by the containers. Each container is supposed to refresh * its current state via heartbeat requests, and the TaskCommunicator implementation is supposed * to aggregate this properly. + * * @return memory in MB */ public long getTotalUsedMemory() { diff --git a/tez-dag/src/main/java/org/apache/tez/serviceplugins/api/TaskCommunicatorContext.java b/tez-dag/src/main/java/org/apache/tez/serviceplugins/api/TaskCommunicatorContext.java index 6741a361c9..014c25fb15 100644 --- a/tez-dag/src/main/java/org/apache/tez/serviceplugins/api/TaskCommunicatorContext.java +++ b/tez-dag/src/main/java/org/apache/tez/serviceplugins/api/TaskCommunicatorContext.java @@ -26,7 +26,6 @@ import org.apache.tez.dag.records.TezTaskAttemptID; import org.apache.tez.runtime.api.TaskFailureType; - // Do not make calls into this from within a held lock. // TODO TEZ-2003 (post) TEZ-2665. Move to the tez-api module @@ -43,7 +42,6 @@ public interface TaskCommunicatorContext extends ServicePluginContextBase { // - Maybe add book-keeping as a helper library, instead of each impl tracking container to task etc. // - Handling of containres / tasks which no longer exist in the system (formalized interface instead of a shouldDie notification) - /** * Get the application attempt id for the running application. Relevant when running under YARN * @@ -127,7 +125,7 @@ public interface TaskCommunicatorContext extends ServicePluginContextBase { /** * Inform the framework that the task has started execution - * + *

    * Use {@link #taskSubmitted(TezTaskAttemptID, ContainerId)} * and {@link #taskStartedRemotely(TezTaskAttemptID)} instead * @@ -176,6 +174,7 @@ void taskFailed(TezTaskAttemptID taskAttemptId, TaskFailureType taskFailureType, /** * Get an identifier for the executing context of the DAG. + * * @return a String identifier for the exeucting context. */ String getCurrentAppIdentifier(); diff --git a/tez-dag/src/main/java/org/apache/tez/serviceplugins/api/TaskHeartbeatRequest.java b/tez-dag/src/main/java/org/apache/tez/serviceplugins/api/TaskHeartbeatRequest.java index 2f9fb7f60c..cbd4157250 100644 --- a/tez-dag/src/main/java/org/apache/tez/serviceplugins/api/TaskHeartbeatRequest.java +++ b/tez-dag/src/main/java/org/apache/tez/serviceplugins/api/TaskHeartbeatRequest.java @@ -30,7 +30,6 @@ public class TaskHeartbeatRequest { private final int preRoutedStartIndex; private final int maxEvents; - public TaskHeartbeatRequest(String containerIdentifier, TezTaskAttemptID taskAttemptId, List events, int startIndex, int preRoutedStartIndex, int maxEvents) { diff --git a/tez-dag/src/main/java/org/apache/tez/state/OnStateChangedCallback.java b/tez-dag/src/main/java/org/apache/tez/state/OnStateChangedCallback.java index 53767fcd70..ae10b9b095 100644 --- a/tez-dag/src/main/java/org/apache/tez/state/OnStateChangedCallback.java +++ b/tez-dag/src/main/java/org/apache/tez/state/OnStateChangedCallback.java @@ -18,7 +18,6 @@ package org.apache.tez.state; - public interface OnStateChangedCallback, OPERAND> { public void onStateChanged(OPERAND operand, STATE state); -} \ No newline at end of file +} diff --git a/tez-dag/src/test/java/org/apache/tez/dag/api/client/TestDAGClientHandler.java b/tez-dag/src/test/java/org/apache/tez/dag/api/client/TestDAGClientHandler.java index c7daebf02d..a47209ad25 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/api/client/TestDAGClientHandler.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/api/client/TestDAGClientHandler.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -38,9 +38,8 @@ import org.mockito.ArgumentCaptor; import org.mockito.internal.util.collections.Sets; - public class TestDAGClientHandler { - + @Test(timeout = 5000) public void testDAGClientHandler() throws TezException { @@ -78,7 +77,7 @@ public void testDAGClientHandler() throws TezException { } catch (TezException e) { assertTrue(e.getMessage().contains("Unknown dagId")); } - DAGStatus dagStatus = dagClientHandler.getDAGStatus("dag_9999_0001_1", + DAGStatus dagStatus = dagClientHandler.getDAGStatus("dag_9999_0001_1", Sets.newSet(StatusGetOpts.GET_COUNTERS)); assertEquals(mockDagStatusBuilder, dagStatus); @@ -92,8 +91,7 @@ public void testDAGClientHandler() throws TezException { VertexStatus vertexStatus = dagClientHandler.getVertexStatus("dag_9999_0001_1", "v1", Sets.newSet(StatusGetOpts.GET_COUNTERS)); assertEquals(mockVertexStatusBuilder, vertexStatus); - - + // getTezAppMasterStatus when(mockDagAM.isSession()).thenReturn(false); @@ -104,12 +102,12 @@ public void testDAGClientHandler() throws TezException { assertEquals(TezAppMasterStatus.INITIALIZING, dagClientHandler.getTezAppMasterStatus()); when(mockDagAM.getState()).thenReturn(DAGAppMasterState.ERROR); assertEquals(TezAppMasterStatus.SHUTDOWN, dagClientHandler.getTezAppMasterStatus()); - + // tryKillDAG - try{ + try { dagClientHandler.tryKillDAG("dag_9999_0001_2"); fail("should not come here"); - }catch(TezException e){ + } catch (TezException e) { assertTrue(e.getMessage().contains("Unknown dagId")); } dagClientHandler.tryKillDAG("dag_9999_0001_1"); @@ -118,17 +116,16 @@ public void testDAGClientHandler() throws TezException { contains("Sending client kill from")); assertEquals(1, eventCaptor.getAllValues().size()); assertTrue(eventCaptor.getAllValues().get(0) instanceof DAG); - assertEquals("dag_9999_0001_1", ((DAG)eventCaptor.getAllValues().get(0)).getID().toString()); + assertEquals("dag_9999_0001_1", ((DAG) eventCaptor.getAllValues().get(0)).getID().toString()); // submitDAG DAGPlan dagPlan = DAGPlan.getDefaultInstance(); - Map localResources = new HashMap(); + Map localResources = new HashMap(); dagClientHandler.submitDAG(dagPlan, localResources); verify(mockDagAM).submitDAGToAppMaster(dagPlan, localResources); - + // shutdown dagClientHandler.shutdownAM(); verify(mockDagAM).shutdownTezAM(contains("Received message to shutdown AM from")); } - } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/api/client/TestDAGClientServer.java b/tez-dag/src/test/java/org/apache/tez/dag/api/client/TestDAGClientServer.java index 06280d8684..ca1b0385ac 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/api/client/TestDAGClientServer.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/api/client/TestDAGClientServer.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-dag/src/test/java/org/apache/tez/dag/api/client/TestVertexStatusBuilder.java b/tez-dag/src/test/java/org/apache/tez/dag/api/client/TestVertexStatusBuilder.java index 0159372270..1aef2fab2c 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/api/client/TestVertexStatusBuilder.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/api/client/TestVertexStatusBuilder.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,5 +36,4 @@ public void testVertexStateConversion() { Assert.assertEquals(state.name(), clientState.name()); } } - } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClientAMProtocolBlockingPBServerImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClientAMProtocolBlockingPBServerImpl.java index 1aea8f793d..7b2a5fc14a 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClientAMProtocolBlockingPBServerImpl.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/api/client/rpc/TestDAGClientAMProtocolBlockingPBServerImpl.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -109,7 +109,7 @@ public void testSubmitDagInSessionWithLargeDagPlan() throws Exception { DAGClientAMProtocolBlockingPBServerImpl serverImpl = spy(new DAGClientAMProtocolBlockingPBServerImpl( dagClientHandler, FileSystem.get(conf))); when(dagClientHandler.getACLManager()).thenReturn(aclManager); - when(dagClientHandler.submitDAG((DAGPlan)any(), (Map)any())).thenReturn("dag-id"); + when(dagClientHandler.submitDAG((DAGPlan) any(), (Map) any())).thenReturn("dag-id"); when(aclManager.checkAMModifyAccess((UserGroupInformation) any())).thenReturn(true); requestBuilder.clear().setSerializedRequestPath(requestFile.getAbsolutePath()); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/MockClock.java b/tez-dag/src/test/java/org/apache/tez/dag/app/MockClock.java index c7cf74eb84..4662995812 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/MockClock.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/MockClock.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app; @@ -24,7 +24,7 @@ import java.util.LinkedList; public class MockClock implements Clock { - + long time; Collection listeners = new LinkedList<>(); @@ -40,7 +40,7 @@ public MockClock(long initTime) { public long getTime() { return time; } - + public void incrementTime(long inc) { time += inc; for (MockClockListener listener : listeners) { diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java index b3ddaa0f86..4c769376c0 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/MockDAGAppMaster.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app; @@ -85,7 +85,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; public class MockDAGAppMaster extends DAGAppMaster { - + private static final Logger LOG = LoggerFactory.getLogger(MockDAGAppMaster.class); MockContainerLauncher containerLauncher; private final AtomicBoolean launcherGoFlag; @@ -100,25 +100,27 @@ public class MockDAGAppMaster extends DAGAppMaster { boolean doSleep = true; int handlerConcurrency = 1; int numConcurrentContainers = 1; - + ThreadMXBean threadMxBean = ManagementFactory.getThreadMXBean(); AtomicLong heartbeatCpu = new AtomicLong(0); AtomicLong heartbeatTime = new AtomicLong(0); AtomicLong numHearbeats = new AtomicLong(0); - + public static interface StatisticsDelegate { public TaskStatistics getStatistics(TaskSpec taskSpec); } + public static interface CountersDelegate { public TezCounters getCounters(TaskSpec taskSpec); } - + public static interface EventsDelegate { public void getEvents(TaskSpec taskSpec, List events, long time); } - + public static interface ContainerDelegate { public void stop(ContainerStopRequest event); + public void launch(ContainerLaunchRequest event); } @@ -131,20 +133,20 @@ public class MockContainerLauncher extends ContainerLauncher implements Runnable BlockingQueue eventQueue = new LinkedBlockingQueue(); Thread eventHandlingThread; ListeningExecutorService executorService; - + Map containers = Maps.newConcurrentMap(); ArrayBlockingQueue workers; TaskCommunicatorManager taskCommunicatorManager; TezTaskCommunicatorImpl taskCommunicator; - + AtomicBoolean startScheduling = new AtomicBoolean(true); AtomicBoolean goFlag; boolean updateProgress = true; LinkedBlockingQueue containersToProcess = new LinkedBlockingQueue(); - + Map preemptedTasks = Maps.newConcurrentMap(); - + Map tasksWithStatusUpdates = Maps.newConcurrentMap(); public MockContainerLauncher(AtomicBoolean goFlag, @@ -153,7 +155,6 @@ public MockContainerLauncher(AtomicBoolean goFlag, this.goFlag = goFlag; } - public class ContainerData { ContainerId cId; TezTaskAttemptID taId; @@ -166,17 +167,17 @@ public class ContainerData { boolean completed; String cIdStr; AtomicBoolean remove = new AtomicBoolean(false); - + public ContainerData(ContainerId cId, ContainerLaunchContext context) { this.cId = cId; this.cIdStr = cId.toString(); this.launchContext = context; } - + void remove() { remove.set(true); } - + void clear() { taId = null; vName = null; @@ -190,7 +191,7 @@ void clear() { remove.set(false); } } - + @Override public void start() throws Exception { taskCommunicatorManager = (TaskCommunicatorManager) getTaskCommunicatorManager(); @@ -201,9 +202,9 @@ public void start() throws Exception { new ThreadFactoryBuilder().setDaemon(true).setNameFormat("MockLauncherExecutionThread [%d]") .build()); this.executorService = MoreExecutors.listeningDecorator(rawExecutor); - int numWorkers = numConcurrentContainers*2; // handle races that cause extra + int numWorkers = numConcurrentContainers * 2; // handle races that cause extra workers = new ArrayBlockingQueue(numWorkers); - for (int i=0; i getContainers() { return containers; } - + public void preemptContainerForTask(TezTaskID tId, int uptoVersion) { preemptedTasks.put(tId, uptoVersion); } - + public void preemptContainer(ContainerData cData) { getTaskSchedulerManager().containerCompleted(0, null, ContainerStatus.newInstance(cData.cId, null, "Preempted", ContainerExitStatus.PREEMPTED)); cData.clear(); } - + public void setStatusUpdatesForTask(TezTaskAttemptID tId, int numUpdates) { tasksWithStatusUpdates.put(tId, numUpdates); } - + void stop(ContainerStopRequest event) { // remove from simulated container list containers.remove(event.getContainerId()); @@ -291,20 +291,20 @@ void launch(ContainerLaunchRequest event) { } getContext().containerLaunched(event.getContainerId()); } - + public void waitTillContainersLaunched() throws InterruptedException { while (containers.isEmpty()) { Thread.sleep(50); } } - + void incrementTime(long inc) { Clock clock = MockDAGAppMaster.this.getContext().getClock(); if (clock instanceof MockClock) { ((MockClock) clock).incrementTime(inc); } } - + @Override public void run() { Thread.currentThread().setName("MockLauncher"); @@ -336,7 +336,7 @@ public void run() { LOG.warn("Exception in mock container launcher thread", ie); } } - + private void doHeartbeat(TezHeartbeatRequest request, ContainerData cData) throws Exception { long startTime = System.nanoTime(); long startCpuTime = threadMxBean.getCurrentThreadCpuTime(); @@ -349,13 +349,13 @@ private void doHeartbeat(TezHeartbeatRequest request, ContainerData cData) throw if (!response.getEvents().isEmpty()) { long stopTime = System.nanoTime(); long stopCpuTime = threadMxBean.getCurrentThreadCpuTime(); - heartbeatTime.addAndGet((stopTime-startTime)/1000); - heartbeatCpu.addAndGet((stopCpuTime-startCpuTime)/1000); + heartbeatTime.addAndGet((stopTime - startTime) / 1000); + heartbeatCpu.addAndGet((stopCpuTime - startCpuTime) / 1000); numHearbeats.incrementAndGet(); } } } - + class Worker implements Callable { class WorkerCallback implements FutureCallback { @Override @@ -378,15 +378,15 @@ void completeOperation() { volatile ContainerData cData; WorkerCallback callback = new WorkerCallback(); - + WorkerCallback getCallback() { return callback; } - + void setContainerData(ContainerData cData) { this.cData = cData; } - + @Override public Void call() throws Exception { try { @@ -411,13 +411,13 @@ public Void call() throws Exception { if (cData.numUpdates == 0 || // do at least one update updatesToMake != null && cData.numUpdates < updatesToMake) { List events = Lists.newArrayListWithCapacity( - cData.taskSpec.getOutputs().size() + 1); + cData.taskSpec.getOutputs().size() + 1); if (cData.numUpdates == 0 && eventsDelegate != null) { eventsDelegate.getEvents(cData.taskSpec, events, MockDAGAppMaster.this.getContext().getClock().getTime()); } TezCounters counters = null; if (countersDelegate != null) { - counters = countersDelegate.getCounters(cData.taskSpec); + counters = countersDelegate.getCounters(cData.taskSpec); } TaskStatistics stats = null; if (statsDelegate != null) { @@ -425,10 +425,10 @@ public Void call() throws Exception { } cData.numUpdates++; float maxUpdates = (updatesToMake != null) ? updatesToMake.intValue() : 1; - float progress = updateProgress ? cData.numUpdates/maxUpdates : 0f; - events.add(new TezEvent(new TaskStatusUpdateEvent(counters, progress, stats, false), + float progress = updateProgress ? cData.numUpdates / maxUpdates : 0f; + events.add(new TezEvent(new TaskStatusUpdateEvent(counters, progress, stats, false), new EventMetaData( - EventProducerConsumerType.SYSTEM, cData.vName, "", cData.taId), + EventProducerConsumerType.SYSTEM, cData.vName, "", cData.taId), MockDAGAppMaster.this.getContext().getClock().getTime())); TezHeartbeatRequest request = new TezHeartbeatRequest(cData.numUpdates, events, cData.nextPreRoutedFromEventId, cData.cIdStr, cData.taId, cData.nextFromEventId, 50000, 0); @@ -440,7 +440,7 @@ public Void call() throws Exception { cData.completed = true; List events = Collections.singletonList(new TezEvent( new TaskAttemptCompletedEvent(), new EventMetaData( - EventProducerConsumerType.SYSTEM, cData.vName, "", cData.taId), + EventProducerConsumerType.SYSTEM, cData.vName, "", cData.taId), MockDAGAppMaster.this.getContext().getClock().getTime())); TezHeartbeatRequest request = new TezHeartbeatRequest(++cData.numUpdates, events, cData.nextPreRoutedFromEventId, cData.cIdStr, cData.taId, cData.nextFromEventId, 10000, 0); @@ -455,10 +455,9 @@ public Void call() throws Exception { } return null; } - } } - + public class MockHistoryEventHandler extends HistoryEventHandler { public MockHistoryEventHandler(AppContext context) { @@ -489,16 +488,15 @@ public boolean wasShutdownInvoked() { return shutdownInvoked.get() > 0 || shutdownInvokedWithoutDelay.get() > 0; } - } public MockDAGAppMaster(ApplicationAttemptId applicationAttemptId, ContainerId containerId, - String nmHost, int nmPort, int nmHttpPort, Clock clock, long appSubmitTime, - boolean isSession, String workingDirectory, String[] localDirs, String[] logDirs, - AtomicBoolean launcherGoFlag, boolean initFailFlag, boolean startFailFlag, - Credentials credentials, String jobUserName, int handlerConcurrency, int numConcurrentContainers) { + String nmHost, int nmPort, int nmHttpPort, Clock clock, long appSubmitTime, + boolean isSession, String workingDirectory, String[] localDirs, String[] logDirs, + AtomicBoolean launcherGoFlag, boolean initFailFlag, boolean startFailFlag, + Credentials credentials, String jobUserName, int handlerConcurrency, int numConcurrentContainers) { super(applicationAttemptId, containerId, nmHost, nmPort, nmHttpPort, clock, appSubmitTime, - isSession, workingDirectory, localDirs, logDirs, new TezApiVersionInfo().getVersion(), + isSession, workingDirectory, localDirs, logDirs, new TezApiVersionInfo().getVersion(), credentials, jobUserName, null); shutdownHandler = new MockDAGAppMasterShutdownHandler(); this.launcherGoFlag = launcherGoFlag; @@ -541,7 +539,7 @@ public MockContainerLauncher getContainerLauncher() { public MockDAGAppMasterShutdownHandler getShutdownHandler() { return (MockDAGAppMasterShutdownHandler) this.shutdownHandler; } - + public void clearStats() { heartbeatCpu.set(0); heartbeatTime.set(0); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/MockLocalClient.java b/tez-dag/src/test/java/org/apache/tez/dag/app/MockLocalClient.java index c3355473d1..cc2eaa9aaf 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/MockLocalClient.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/MockLocalClient.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app; @@ -40,7 +40,7 @@ public MockLocalClient(AtomicBoolean mockAppLauncherGoFlag, Clock clock) { } public MockLocalClient(AtomicBoolean mockAppLauncherGoFlag, Clock clock, - boolean initFailFlag, boolean startFailFlag, int concurrency, int containers) { + boolean initFailFlag, boolean startFailFlag, int concurrency, int containers) { this.mockAppLauncherGoFlag = mockAppLauncherGoFlag; this.mockClock = clock; this.initFailFlag = initFailFlag; @@ -51,12 +51,12 @@ public MockLocalClient(AtomicBoolean mockAppLauncherGoFlag, Clock clock, @Override protected DAGAppMaster createDAGAppMaster(ApplicationAttemptId applicationAttemptId, - ContainerId cId, String currentHost, int nmPort, int nmHttpPort, - Clock clock, long appSubmitTime, boolean isSession, String userDir, - String[] localDirs, String[] logDirs, Credentials credentials, String jobUserName) { + ContainerId cId, String currentHost, int nmPort, int nmHttpPort, + Clock clock, long appSubmitTime, boolean isSession, String userDir, + String[] localDirs, String[] logDirs, Credentials credentials, String jobUserName) { mockApp = new MockDAGAppMaster(applicationAttemptId, cId, currentHost, nmPort, nmHttpPort, - (mockClock!=null ? mockClock : clock), appSubmitTime, isSession, userDir, localDirs, logDirs, - mockAppLauncherGoFlag, initFailFlag, startFailFlag, credentials, jobUserName, + (mockClock != null ? mockClock : clock), appSubmitTime, isSession, userDir, localDirs, logDirs, + mockAppLauncherGoFlag, initFailFlag, startFailFlag, credentials, jobUserName, concurrency, containers); return mockApp; } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/MockTezClient.java b/tez-dag/src/test/java/org/apache/tez/dag/app/MockTezClient.java index ff6697026c..3ae8caac9d 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/MockTezClient.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/MockTezClient.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app; @@ -30,37 +30,36 @@ public class MockTezClient extends TezClient { MockLocalClient client; - + MockTezClient(String name, TezConfiguration tezConf, boolean isSession, - Map localResources, Credentials credentials, - Clock clock, AtomicBoolean mockAppLauncherGoFlag) { + Map localResources, Credentials credentials, + Clock clock, AtomicBoolean mockAppLauncherGoFlag) { super(name, tezConf, isSession, localResources, credentials); this.client = new MockLocalClient(mockAppLauncherGoFlag, clock); } - + MockTezClient(String name, TezConfiguration tezConf, boolean isSession, - Map localResources, Credentials credentials, - Clock clock, AtomicBoolean mockAppLauncherGoFlag, - boolean initFailFlag, boolean startFailFlag) { + Map localResources, Credentials credentials, + Clock clock, AtomicBoolean mockAppLauncherGoFlag, + boolean initFailFlag, boolean startFailFlag) { this(name, tezConf, isSession, localResources, credentials, clock, mockAppLauncherGoFlag, initFailFlag, startFailFlag, 1, 1); } - + MockTezClient(String name, TezConfiguration tezConf, boolean isSession, - Map localResources, Credentials credentials, - Clock clock, AtomicBoolean mockAppLauncherGoFlag, - boolean initFailFlag, boolean startFailFlag, int concurrency, int containers) { + Map localResources, Credentials credentials, + Clock clock, AtomicBoolean mockAppLauncherGoFlag, + boolean initFailFlag, boolean startFailFlag, int concurrency, int containers) { super(name, tezConf, isSession, localResources, credentials); - this.client = new MockLocalClient(mockAppLauncherGoFlag, clock, initFailFlag, startFailFlag, + this.client = new MockLocalClient(mockAppLauncherGoFlag, clock, initFailFlag, startFailFlag, concurrency, containers); } protected FrameworkClient createFrameworkClient() { return client; } - + public MockLocalClient getLocalClient() { return client; } - } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/PluginWrapperTestHelpers.java b/tez-dag/src/test/java/org/apache/tez/dag/app/PluginWrapperTestHelpers.java index fb6faa1bb6..ea7b0720cb 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/PluginWrapperTestHelpers.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/PluginWrapperTestHelpers.java @@ -29,7 +29,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class PluginWrapperTestHelpers { private static final Logger LOG = LoggerFactory.getLogger(PluginWrapperTestHelpers.class); @@ -70,8 +69,6 @@ public static void testDelegation(Class delegateClass, Class rawClass, } } } - - } public static Object[] constructMethodArgs(Method method) throws IllegalAccessException, diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestDAGAppMaster.java index d8167dbcc4..94adb28422 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestDAGAppMaster.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestDAGAppMaster.java @@ -18,6 +18,7 @@ import org.apache.tez.dag.app.dag.DAGState; import org.apache.tez.dag.app.dag.Vertex; import org.apache.tez.dag.records.TezVertexID; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @@ -177,7 +178,6 @@ public void testPluginParsing() throws IOException { assertTrue(pluginMap.containsKey(TezConstants.getTezUberServicePluginName())); assertTrue(1 == pluginMap.get(TezConstants.getTezUberServicePluginName())); - String pluginName = "d1"; ByteBuffer bb = ByteBuffer.allocate(4); bb.putInt(0, 3); @@ -213,7 +213,6 @@ public void testPluginParsing() throws IOException { entityDescriptors.clear(); } - @Test(timeout = 5000) public void testParseAllPluginsNoneSpecified() throws IOException { Configuration conf = new Configuration(false); @@ -227,7 +226,6 @@ public void testParseAllPluginsNoneSpecified() throws IOException { List tcDescriptors; BiMap tcMap; - // No plugins. Non local tsDescriptors = Lists.newLinkedList(); tsMap = HashBiMap.create(); @@ -272,7 +270,6 @@ public void testParseAllPluginsOnlyCustomSpecified() throws IOException { List tcDescriptors; BiMap tcMap; - // Only plugin, Yarn. tsDescriptors = Lists.newLinkedList(); tsMap = HashBiMap.create(); @@ -308,7 +305,6 @@ public void testParseAllPluginsCustomAndYarnSpecified() throws IOException { List tcDescriptors; BiMap tcMap; - // Only plugin, Yarn. tsDescriptors = Lists.newLinkedList(); tsMap = HashBiMap.create(); @@ -358,12 +354,12 @@ private AMPluginDescriptorProto createAmPluginDescriptor(boolean enableYarn, boo .setContainersEnabled(enableYarn); if (addCustom) { builder.addTaskSchedulers( - TezNamedEntityDescriptorProto.newBuilder() - .setName(TS_NAME) - .setEntityDescriptor( - DAGProtos.TezEntityDescriptorProto.newBuilder() - .setClassName(TS_NAME + CLASS_SUFFIX) - .setTezUserPayload(payloadProto))) + TezNamedEntityDescriptorProto.newBuilder() + .setName(TS_NAME) + .setEntityDescriptor( + DAGProtos.TezEntityDescriptorProto.newBuilder() + .setClassName(TS_NAME + CLASS_SUFFIX) + .setTezUserPayload(payloadProto))) .addContainerLaunchers( TezNamedEntityDescriptorProto.newBuilder() .setName(CL_NAME) @@ -382,7 +378,6 @@ private AMPluginDescriptorProto createAmPluginDescriptor(boolean enableYarn, boo return builder.build(); } - @Test public void testDagCredentialsWithoutMerge() throws Exception { testDagCredentials(false); @@ -427,8 +422,8 @@ public void testBadProgress() throws Exception { DAGAppMaster am = spy(new DAGAppMaster(attemptId, ContainerId.newContainerId(attemptId, 1), "127.0.0.1", 0, 0, new MonotonicClock(), 1, true, - TEST_DIR.toString(), new String[] {TEST_DIR.toString()}, - new String[] {TEST_DIR.toString()}, + TEST_DIR.toString(), new String[]{TEST_DIR.toString()}, + new String[]{TEST_DIR.toString()}, new TezApiVersionInfo().getVersion(), amCreds, "someuser", null)); when(am.getState()).thenReturn(DAGAppMasterState.RUNNING); @@ -512,8 +507,8 @@ private void testDagCredentials(boolean doMerge) throws IOException { DAGAppMaster am = new DAGAppMaster(attemptId, ContainerId.newInstance(attemptId, 1), "127.0.0.1", 0, 0, new SystemClock(), 1, true, - TEST_DIR.toString(), new String[] {TEST_DIR.toString()}, - new String[] {TEST_DIR.toString()}, + TEST_DIR.toString(), new String[]{TEST_DIR.toString()}, + new String[]{TEST_DIR.toString()}, new TezApiVersionInfo().getVersion(), amCreds, "someuser", null); am.init(conf); @@ -634,7 +629,7 @@ public static class DAGAppMasterForTest extends DAGAppMaster { public DAGAppMasterForTest(ApplicationAttemptId attemptId, boolean isSession) { super(attemptId, ContainerId.newContainerId(attemptId, 1), "hostname", 12345, 12346, new SystemClock(), 0, isSession, TEST_DIR.getAbsolutePath(), - new String[] { TEST_DIR.getAbsolutePath() }, new String[] { TEST_DIR.getAbsolutePath() }, + new String[]{TEST_DIR.getAbsolutePath()}, new String[]{TEST_DIR.getAbsolutePath()}, new TezDagVersionInfo().getVersion(), createCredentials(), "jobname", null); } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestMemoryWithEvents.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestMemoryWithEvents.java index 535e69ddca..750ea3ad71 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestMemoryWithEvents.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestMemoryWithEvents.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app; @@ -62,7 +62,7 @@ public class TestMemoryWithEvents { static Configuration defaultConf; static FileSystem localFs; - + static { try { defaultConf = new Configuration(false); @@ -81,7 +81,7 @@ public class TestMemoryWithEvents { final int numTasks = 10000; private void checkMemory(String name, MockDAGAppMaster mockApp) { - long mb = 1024*1024; + long mb = 1024 * 1024; long microsPerMs = 1000; //Getting the runtime reference from system @@ -90,7 +90,7 @@ private void checkMemory(String name, MockDAGAppMaster mockApp) { System.out.println("##### Heap utilization statistics [MB] for " + name); runtime.gc(); - + //Print used memory System.out.println("##### Used Memory:" + (runtime.totalMemory() - runtime.freeMemory()) / mb); @@ -98,25 +98,25 @@ private void checkMemory(String name, MockDAGAppMaster mockApp) { //Print free memory System.out.println("##### Free Memory:" + runtime.freeMemory() / mb); - + //Print total available memory System.out.println("##### Total Memory:" + runtime.totalMemory() / mb); //Print Maximum available memory System.out.println("##### Max Memory:" + runtime.maxMemory() / mb); - + //Print Maximum heartbeat time long numHeartbeats = mockApp.numHearbeats.get(); if (numHeartbeats == 0) { numHeartbeats = 1; } - System.out.println("##### Heartbeat (ms) :" - + " latency avg: " + ((mockApp.heartbeatTime.get() / numHeartbeats) / microsPerMs) + System.out.println("##### Heartbeat (ms) :" + + " latency avg: " + ((mockApp.heartbeatTime.get() / numHeartbeats) / microsPerMs) + " cpu total: " + (mockApp.heartbeatCpu.get() / microsPerMs) + " cpu avg: " + ((mockApp.heartbeatCpu.get() / numHeartbeats) / microsPerMs) + " numHeartbeats: " + mockApp.numHearbeats.get()); } - + private void testMemory(DAG dag, boolean sendDMEvents) throws Exception { StopWatch stopwatch = new StopWatch(); stopwatch.start(); @@ -125,7 +125,7 @@ private void testMemory(DAG dag, boolean sendDMEvents) throws Exception { MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false, numThreads, 1000); tezClient.start(); - + MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp(); MockContainerLauncher mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(false); @@ -141,7 +141,7 @@ private void testMemory(DAG dag, boolean sendDMEvents) throws Exception { System.out.println("Time taken(ms): " + stopwatch.now(TimeUnit.MILLISECONDS)); tezClient.stop(); } - + public static class SimulationInitializer extends InputInitializer { public SimulationInitializer(InputInitializerContext initializerContext) { super(initializerContext); @@ -151,7 +151,7 @@ public SimulationInitializer(InputInitializerContext initializerContext) { public List initialize() throws Exception { int numTasks = getContext().getNumTasks(); List events = Lists.newArrayListWithCapacity(numTasks); - for (int i=0; i events) thro } @Ignore - @Test (timeout = 600000) + @Test(timeout = 600000) public void testMemoryRootInputEvents() throws Exception { DAG dag = DAG.create("testMemoryRootInputEvents"); Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), numTasks); @@ -175,9 +175,9 @@ public void testMemoryRootInputEvents() throws Exception { dag.addVertex(vA).addVertex(vB); testMemory(dag, false); } - + @Ignore - @Test (timeout = 600000) + @Test(timeout = 600000) public void testMemoryOneToOne() throws Exception { DAG dag = DAG.create("testMemoryOneToOne"); Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), numTasks); @@ -192,7 +192,7 @@ public void testMemoryOneToOne() throws Exception { } @Ignore - @Test (timeout = 600000) + @Test(timeout = 600000) public void testMemoryBroadcast() throws Exception { DAG dag = DAG.create("testMemoryBroadcast"); Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), numTasks); @@ -205,9 +205,9 @@ public void testMemoryBroadcast() throws Exception { OutputDescriptor.create("Out"), InputDescriptor.create("In")))); testMemory(dag, true); } - + @Ignore - @Test (timeout = 600000) + @Test(timeout = 600000) public void testMemoryScatterGather() throws Exception { DAG dag = DAG.create("testMemoryScatterGather"); Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), numTasks); @@ -220,5 +220,4 @@ public void testMemoryScatterGather() throws Exception { OutputDescriptor.create("Out"), InputDescriptor.create("In")))); testMemory(dag, true); } - } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java index 6c0231c9b0..2ddc812df0 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app; @@ -117,7 +117,8 @@ public class TestMockDAGAppMaster { private static final Log LOG = LogFactory.getLog(TestMockDAGAppMaster.class); static Configuration defaultConf; - static FileSystem localFs; + static FileSystem localFs; + static { try { defaultConf = new Configuration(false); @@ -130,7 +131,7 @@ public class TestMockDAGAppMaster { throw new RuntimeException("init failure", e); } } - + static class TestEventsDelegate implements EventsDelegate { @Override public void getEvents(TaskSpec taskSpec, List events, long time) { @@ -138,28 +139,28 @@ public void getEvents(TaskSpec taskSpec, List events, long time) { if (output.getPhysicalEdgeCount() == 1) { events.add(new TezEvent(DataMovementEvent.create(0, 0, 0, null), new EventMetaData( EventProducerConsumerType.OUTPUT, taskSpec.getVertexName(), output - .getDestinationVertexName(), taskSpec.getTaskAttemptID()), time)); + .getDestinationVertexName(), taskSpec.getTaskAttemptID()), time)); } else { events.add(new TezEvent(CompositeDataMovementEvent.create(0, output.getPhysicalEdgeCount(), null), new EventMetaData( EventProducerConsumerType.OUTPUT, taskSpec.getVertexName(), output - .getDestinationVertexName(), taskSpec.getTaskAttemptID()), time)); + .getDestinationVertexName(), taskSpec.getTaskAttemptID()), time)); } } - } + } } - - @Test (timeout = 5000) + + @Test(timeout = 5000) public void testLocalResourceSetup() throws Exception { TezConfiguration tezconf = new TezConfiguration(defaultConf); - + MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null); tezClient.start(); - + MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp(); MockContainerLauncher mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(false); - + Map lrDAG = Maps.newHashMap(); String lrName1 = "LR1"; lrDAG.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), @@ -181,20 +182,20 @@ public void testLocalResourceSetup() throws Exception { // verify tasks are launched with both DAG and task resources. Assert.assertTrue(taskLR.containsKey(lrName1)); Assert.assertTrue(taskLR.containsKey(lrName2)); - + mockLauncher.startScheduling(true); dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState()); tezClient.stop(); } - - @Test (timeout = 5000) + + @Test(timeout = 5000) public void testInternalPreemption() throws Exception { TezConfiguration tezconf = new TezConfiguration(defaultConf); - + MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null); tezClient.start(); - + MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp(); MockContainerLauncher mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(false); @@ -208,7 +209,7 @@ public void testInternalPreemption() throws Exception { ContainerData cData = mockLauncher.getContainers().values().iterator().next(); DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG(); mockApp.getTaskSchedulerManager().preemptContainer(0, cData.cId); - + mockLauncher.startScheduling(true); dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState()); @@ -220,13 +221,13 @@ public void testInternalPreemption() throws Exception { tezClient.stop(); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testBasicEvents() throws Exception { TezConfiguration tezconf = new TezConfiguration(defaultConf); - + MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null); tezClient.start(); - + MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp(); MockContainerLauncher mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(false); @@ -265,11 +266,11 @@ public void testBasicEvents() throws Exception { List tEvents = vImpl.getTaskAttemptTezEvents(taId, 0, 0, 1000).getEvents(); Assert.assertEquals(2, tEvents.size()); // 2 from vA Assert.assertEquals(vA.getName(), tEvents.get(0).getDestinationInfo().getEdgeVertexName()); - Assert.assertEquals(0, ((DataMovementEvent)tEvents.get(0).getEvent()).getSourceIndex()); + Assert.assertEquals(0, ((DataMovementEvent) tEvents.get(0).getEvent()).getSourceIndex()); Assert.assertEquals(vA.getName(), tEvents.get(1).getDestinationInfo().getEdgeVertexName()); - Assert.assertEquals(0, ((DataMovementEvent)tEvents.get(1).getEvent()).getSourceIndex()); - int targetIndex1 = ((DataMovementEvent)tEvents.get(0).getEvent()).getTargetIndex(); - int targetIndex2 = ((DataMovementEvent)tEvents.get(1).getEvent()).getTargetIndex(); + Assert.assertEquals(0, ((DataMovementEvent) tEvents.get(1).getEvent()).getSourceIndex()); + int targetIndex1 = ((DataMovementEvent) tEvents.get(0).getEvent()).getTargetIndex(); + int targetIndex2 = ((DataMovementEvent) tEvents.get(1).getEvent()).getTargetIndex(); // order of vA task completion can change order of events Assert.assertTrue("t1: " + targetIndex1 + " t2: " + targetIndex2, (targetIndex1 == 0 && targetIndex2 == 1) || (targetIndex1 == 1 && targetIndex2 == 0)); @@ -279,11 +280,11 @@ public void testBasicEvents() throws Exception { tEvents = vImpl.getTaskAttemptTezEvents(taId, 0, 0, 1000).getEvents(); Assert.assertEquals(2, tEvents.size()); // 2 from vA Assert.assertEquals(vA.getName(), tEvents.get(0).getDestinationInfo().getEdgeVertexName()); - Assert.assertEquals(1, ((CompositeRoutedDataMovementEvent)tEvents.get(0).getEvent()).getSourceIndex()); + Assert.assertEquals(1, ((CompositeRoutedDataMovementEvent) tEvents.get(0).getEvent()).getSourceIndex()); Assert.assertEquals(vA.getName(), tEvents.get(1).getDestinationInfo().getEdgeVertexName()); - Assert.assertEquals(1, ((CompositeRoutedDataMovementEvent)tEvents.get(1).getEvent()).getSourceIndex()); - targetIndex1 = ((CompositeRoutedDataMovementEvent)tEvents.get(0).getEvent()).getTargetIndex(); - targetIndex2 = ((CompositeRoutedDataMovementEvent)tEvents.get(1).getEvent()).getTargetIndex(); + Assert.assertEquals(1, ((CompositeRoutedDataMovementEvent) tEvents.get(1).getEvent()).getSourceIndex()); + targetIndex1 = ((CompositeRoutedDataMovementEvent) tEvents.get(0).getEvent()).getTargetIndex(); + targetIndex2 = ((CompositeRoutedDataMovementEvent) tEvents.get(1).getEvent()).getTargetIndex(); // order of vA task completion can change order of events Assert.assertTrue("t1: " + targetIndex1 + " t2: " + targetIndex2, (targetIndex1 == 0 && targetIndex2 == 1) || (targetIndex1 == 1 && targetIndex2 == 0)); @@ -293,14 +294,15 @@ public void testBasicEvents() throws Exception { tEvents = vImpl.getTaskAttemptTezEvents(taId, 0, 0, 1000).getEvents(); Assert.assertEquals(1, tEvents.size()); // 1 from vA Assert.assertEquals(vA.getName(), tEvents.get(0).getDestinationInfo().getEdgeVertexName()); - Assert.assertEquals(0, ((DataMovementEvent)tEvents.get(0).getEvent()).getTargetIndex()); - Assert.assertEquals(0, ((DataMovementEvent)tEvents.get(0).getEvent()).getSourceIndex()); + Assert.assertEquals(0, ((DataMovementEvent) tEvents.get(0).getEvent()).getTargetIndex()); + Assert.assertEquals(0, ((DataMovementEvent) tEvents.get(0).getEvent()).getSourceIndex()); tezClient.stop(); } - + public static class LegacyEdgeTestEdgeManager extends EdgeManagerPlugin { List destinationInputIndices = Collections.singletonList(0); + public LegacyEdgeTestEdgeManager(EdgeManagerPluginContext context) { super(context); } @@ -321,36 +323,36 @@ public int getNumSourceTaskPhysicalOutputs(int sourceTaskIndex) throws Exception @Override public void routeDataMovementEventToDestination(DataMovementEvent event, - int sourceTaskIndex, int sourceOutputIndex, - Map> destinationTaskAndInputIndices) { + int sourceTaskIndex, int sourceOutputIndex, + Map> destinationTaskAndInputIndices) { destinationTaskAndInputIndices.put(sourceTaskIndex, destinationInputIndices); } @Override public void routeInputSourceTaskFailedEventToDestination(int sourceTaskIndex, - Map> destinationTaskAndInputIndices) { + Map> destinationTaskAndInputIndices) { destinationTaskAndInputIndices.put(sourceTaskIndex, destinationInputIndices); } @Override public int routeInputErrorEventToSource(InputReadErrorEvent event, - int destinationTaskIndex, int destinationFailedInputIndex) { + int destinationTaskIndex, int destinationFailedInputIndex) { return destinationTaskIndex; } - + @Override public int getNumDestinationConsumerTasks(int sourceTaskIndex) { return 1; } } - - @Test (timeout = 100000) + + @Test(timeout = 100000) public void testMixedEdgeRouting() throws Exception { - TezConfiguration tezconf = new TezConfiguration(defaultConf); - + TezConfiguration tezconf = new TezConfiguration(defaultConf); + MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null); tezClient.start(); - + MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp(); MockContainerLauncher mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(false); @@ -385,9 +387,9 @@ public void testMixedEdgeRouting() throws Exception { OutputDescriptor.create("Out"), InputDescriptor.create("In")))) .addEdge( Edge.create(vB, vE, EdgeProperty.create( - EdgeManagerPluginDescriptor.create(LegacyEdgeTestEdgeManager.class.getName()), - DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, - OutputDescriptor.create("Out"), InputDescriptor.create("In")))); + EdgeManagerPluginDescriptor.create(LegacyEdgeTestEdgeManager.class.getName()), + DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, + OutputDescriptor.create("Out"), InputDescriptor.create("In")))); DAGClient dagClient = tezClient.submitDAG(dag); mockLauncher.waitTillContainersLaunched(); @@ -413,23 +415,23 @@ public void testMixedEdgeRouting() throws Exception { tezClient.stop(); } - - @Test (timeout = 100000) + + @Test(timeout = 100000) public void testConcurrencyLimit() throws Exception { // the test relies on local mode behavior of launching a new container per task. // so task concurrency == container concurrency TezConfiguration tezconf = new TezConfiguration(defaultConf); - + final int concurrencyLimit = 5; MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, - null, false, false, concurrencyLimit*4, 1000); + null, false, false, concurrencyLimit * 4, 1000); tezClient.start(); - + MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp(); MockContainerLauncher mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(false); - + final AtomicInteger concurrency = new AtomicInteger(0); final AtomicBoolean exceededConcurrency = new AtomicBoolean(false); mockApp.containerDelegate = new ContainerDelegate() { @@ -437,6 +439,7 @@ public void testConcurrencyLimit() throws Exception { public void stop(ContainerStopRequest event) { concurrency.decrementAndGet(); } + @Override public void launch(ContainerLaunchRequest event) { int maxConc = concurrency.incrementAndGet(); @@ -463,7 +466,7 @@ public void launch(ContainerLaunchRequest event) { public void testCountersAggregation() throws Exception { TezConfiguration tezconf = new TezConfiguration(defaultConf); MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, - null, false, false); + null, false, false); tezClient.start(); final String vAName = "A"; @@ -477,8 +480,8 @@ public void testCountersAggregation() throws Exception { .addVertex(vB) .addEdge( Edge.create(vA, vB, EdgeProperty.create(DataMovementType.SCATTER_GATHER, - DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, - OutputDescriptor.create("Out"), InputDescriptor.create("In")))); + DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, + OutputDescriptor.create("Out"), InputDescriptor.create("In")))); TezCounters temp = new TezCounters(); temp.findCounter(new String(globalCounterName), new String(globalCounterName)).increment(1); ByteArrayOutputStream bos = new ByteArrayOutputStream(); @@ -491,11 +494,12 @@ public void testCountersAggregation() throws Exception { mockLauncher.startScheduling(false); mockApp.countersDelegate = new CountersDelegate() { int counterValue = 0; + @Override public TezCounters getCounters(TaskSpec taskSpec) { String vName = taskSpec.getVertexName(); TezCounters counters = new TezCounters(); - final DataInputByteBuffer in = new DataInputByteBuffer(); + final DataInputByteBuffer in = new DataInputByteBuffer(); in.reset(ByteBuffer.wrap(payload)); try { // this ensures that the serde code path is covered. @@ -529,20 +533,20 @@ public TezCounters getCounters(TaskSpec taskSpec) { TezCounters vACounters = vAImpl.getAllCounters(); TezCounters vBCounters = vBImpl.getAllCounters(); - Assert.assertEquals(19, ((AggregateTezCounterDelegate)vACounters.findCounter(vAName, procCounterName)).getMax()); - Assert.assertEquals(1, ((AggregateTezCounterDelegate)vACounters.findCounter(vAName, procCounterName)).getMin()); - Assert.assertEquals(20, ((AggregateTezCounterDelegate)vACounters.findCounter(vAName, vBName)).getMax()); - Assert.assertEquals(2, ((AggregateTezCounterDelegate)vACounters.findCounter(vAName, vBName)).getMin()); + Assert.assertEquals(19, ((AggregateTezCounterDelegate) vACounters.findCounter(vAName, procCounterName)).getMax()); + Assert.assertEquals(1, ((AggregateTezCounterDelegate) vACounters.findCounter(vAName, procCounterName)).getMin()); + Assert.assertEquals(20, ((AggregateTezCounterDelegate) vACounters.findCounter(vAName, vBName)).getMax()); + Assert.assertEquals(2, ((AggregateTezCounterDelegate) vACounters.findCounter(vAName, vBName)).getMin()); - Assert.assertEquals(21, ((AggregateTezCounterDelegate)vBCounters.findCounter(vBName, procCounterName)).getMin()); - Assert.assertEquals(21, ((AggregateTezCounterDelegate)vBCounters.findCounter(vBName, procCounterName)).getMax()); - Assert.assertEquals(22, ((AggregateTezCounterDelegate)vBCounters.findCounter(vBName, vAName)).getMin()); - Assert.assertEquals(22, ((AggregateTezCounterDelegate)vBCounters.findCounter(vBName, vAName)).getMax()); + Assert.assertEquals(21, ((AggregateTezCounterDelegate) vBCounters.findCounter(vBName, procCounterName)).getMin()); + Assert.assertEquals(21, ((AggregateTezCounterDelegate) vBCounters.findCounter(vBName, procCounterName)).getMax()); + Assert.assertEquals(22, ((AggregateTezCounterDelegate) vBCounters.findCounter(vBName, vAName)).getMin()); + Assert.assertEquals(22, ((AggregateTezCounterDelegate) vBCounters.findCounter(vBName, vAName)).getMax()); tezClient.stop(); } - @Test (timeout = 10000) + @Test(timeout = 10000) public void testBasicCounters() throws Exception { TezConfiguration tezconf = new TezConfiguration(defaultConf); MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, @@ -577,7 +581,7 @@ public void testBasicCounters() throws Exception { public TezCounters getCounters(TaskSpec taskSpec) { String vName = taskSpec.getVertexName(); TezCounters counters = new TezCounters(); - final DataInputByteBuffer in = new DataInputByteBuffer(); + final DataInputByteBuffer in = new DataInputByteBuffer(); in.reset(ByteBuffer.wrap(payload)); try { // this ensures that the serde code path is covered. @@ -634,11 +638,11 @@ public TezCounters getCounters(TaskSpec taskSpec) { if (vaGrouName != vBGrouName) { Assert.fail("String group name objects dont match despite interning."); } - + tezClient.stop(); } - - @Test (timeout = 10000) + + @Test(timeout = 10000) public void testBasicStatistics() throws Exception { TezConfiguration tezconf = new TezConfiguration(defaultConf); MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, @@ -678,7 +682,7 @@ public void testBasicStatistics() throws Exception { DataOutput outB = new DataOutputStream(bosB); vBStats.write(outB); final byte[] payloadB = bosB.toByteArray(); - + MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp(); MockContainerLauncher mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(false); @@ -708,7 +712,7 @@ public TaskStatistics getStatistics(TaskSpec taskSpec) { mockLauncher.startScheduling(true); DAGStatus status = dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState()); - + // verify that the values have been correct aggregated for (org.apache.tez.dag.app.dag.Vertex v : dagImpl.getVertices().values()) { VertexStatistics vStats = v.getStatistics(); @@ -724,37 +728,37 @@ public TaskStatistics getStatistics(TaskSpec taskSpec) { Assert.assertEquals(2, vStats.getOutputStatistics(sinkName).getItemsProcessed()); } } - + tezClient.stop(); } - - private void checkMemory(String name, MockDAGAppMaster mockApp) { - long mb = 1024*1024; - + + private void checkMemory(String name, MockDAGAppMaster mockApp) { + long mb = 1024 * 1024; + //Getting the runtime reference from system - Runtime runtime = Runtime.getRuntime(); - - System.out.println("##### Heap utilization statistics [MB] for " + name); - - runtime.gc(); - + Runtime runtime = Runtime.getRuntime(); + + System.out.println("##### Heap utilization statistics [MB] for " + name); + + runtime.gc(); + //Print used memory - System.out.println("##### Used Memory:" - + (runtime.totalMemory() - runtime.freeMemory()) / mb); - + System.out.println("##### Used Memory:" + + (runtime.totalMemory() - runtime.freeMemory()) / mb); + //Print free memory - System.out.println("##### Free Memory:" - + runtime.freeMemory() / mb); - + System.out.println("##### Free Memory:" + + runtime.freeMemory() / mb); + //Print total available memory - System.out.println("##### Total Memory:" + runtime.totalMemory() / mb); - + System.out.println("##### Total Memory:" + runtime.totalMemory() / mb); + //Print Maximum available memory - System.out.println("##### Max Memory:" + runtime.maxMemory() / mb); - } + System.out.println("##### Max Memory:" + runtime.maxMemory() / mb); + } @Ignore - @Test (timeout = 60000) + @Test(timeout = 60000) public void testBasicCounterMemory() throws Exception { Logger.getRootLogger().setLevel(Level.WARN); TezConfiguration tezconf = new TezConfiguration(defaultConf); @@ -763,7 +767,7 @@ public void testBasicCounterMemory() throws Exception { tezClient.start(); final String vAName = "A"; - + DAG dag = DAG.create("testBasicCounterMemory"); Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), 10000); dag.addVertex(vA); @@ -777,8 +781,8 @@ public TezCounters getCounters(TaskSpec taskSpec) { TezCounters counters = new TezCounters(); final String longName = "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz"; final String shortName = "abcdefghijklmnopqrstuvwxyz"; - for (int i=0; i<6; ++i) { - for (int j=0; j<15; ++j) { + for (int i = 0; i < 6; ++i) { + for (int j = 0; j < 15; ++j) { counters.findCounter((i + longName), (i + (shortName))).increment(1); } } @@ -797,9 +801,9 @@ public TezCounters getCounters(TaskSpec taskSpec) { checkMemory(dag.getName(), mockApp); tezClient.stop(); } - + @Ignore - @Test (timeout = 60000) + @Test(timeout = 60000) public void testTaskEventsProcessingSpeed() throws Exception { Logger.getRootLogger().setLevel(Level.WARN); TezConfiguration tezconf = new TezConfiguration(defaultConf); @@ -809,7 +813,7 @@ public void testTaskEventsProcessingSpeed() throws Exception { tezClient.start(); final String vAName = "A"; - + DAG dag = DAG.create("testTaskEventsProcessingSpeed"); Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), 50000); dag.addVertex(vA); @@ -823,7 +827,7 @@ public void testTaskEventsProcessingSpeed() throws Exception { } @Ignore - @Test (timeout = 60000) + @Test(timeout = 60000) public void testBasicStatisticsMemory() throws Exception { Logger.getRootLogger().setLevel(Level.WARN); TezConfiguration tezconf = new TezConfiguration(defaultConf); @@ -842,7 +846,7 @@ public void testBasicStatisticsMemory() throws Exception { DAG dag = DAG.create("testBasicStatisticsMemory"); Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), numTasks); - for (int i=0; i lrDAG = Maps.newHashMap(); String lrName1 = "LR1"; @@ -905,14 +909,14 @@ public void testMultipleSubmissions() throws Exception { dag.addVertex(vA); TezConfiguration tezconf = new TezConfiguration(defaultConf); - + MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null); tezClient.start(); DAGClient dagClient = tezClient.submitDAG(dag); dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState()); tezClient.stop(); - + // submit the same DAG again to verify it can be done. tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null); tezClient.start(); @@ -922,7 +926,7 @@ public void testMultipleSubmissions() throws Exception { tezClient.stop(); } - @Test (timeout = 10000) + @Test(timeout = 10000) public void testSchedulerErrorHandling() throws Exception { TezConfiguration tezconf = new TezConfiguration(defaultConf); @@ -942,17 +946,17 @@ public void testSchedulerErrorHandling() throws Exception { mockApp.handle(new DAGAppMasterEventSchedulingServiceError( org.apache.hadoop.util.StringUtils.stringifyException(new RuntimeException("Mock error")))); - while(!mockApp.getShutdownHandler().wasShutdownInvoked()) { + while (!mockApp.getShutdownHandler().wasShutdownInvoked()) { Thread.sleep(100); } Assert.assertEquals(DAGState.RUNNING, mockApp.getContext().getCurrentDAG().getState()); } - @Test (timeout = 10000) + @Test(timeout = 10000) public void testInitFailed() throws Exception { TezConfiguration tezconf = new TezConfiguration(defaultConf); MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, - null, null, null, new AtomicBoolean(false), true, false); + null, null, null, new AtomicBoolean(false), true, false); try { tezClient.start(); } catch (Exception e) { @@ -964,11 +968,11 @@ public void testInitFailed() throws Exception { } } - @Test (timeout = 10000) + @Test(timeout = 10000) public void testStartFailed() { TezConfiguration tezconf = new TezConfiguration(defaultConf); MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, - null, null, null, new AtomicBoolean(false), false, true); + null, null, null, new AtomicBoolean(false), false, true); try { tezClient.start(); } catch (Exception e) { @@ -980,7 +984,6 @@ public void testStartFailed() { } } - private OutputCommitterDescriptor createOutputCommitterDesc(boolean failOnCommit) { OutputCommitterDescriptor outputCommitterDesc = OutputCommitterDescriptor.create(FailingOutputCommitter.class.getName()); @@ -1010,13 +1013,13 @@ private DAG createDAG(String dagName, boolean uv12CommitFail, boolean v3CommitFa InputDescriptor.create("dummy input class")), InputDescriptor .create("merge.class")); dag.addVertex(v1) - .addVertex(v2) - .addVertex(v3) - .addEdge(e1); + .addVertex(v2) + .addVertex(v3) + .addEdge(e1); return dag; } - @Test (timeout = 60000) + @Test(timeout = 60000) public void testCommitOutputOnDAGSuccess() throws Exception { TezConfiguration tezconf = new TezConfiguration(defaultConf); MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null); @@ -1027,14 +1030,14 @@ public void testCommitOutputOnDAGSuccess() throws Exception { DAGClient dagClient = tezClient.submitDAG(dag1); dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState()); - + // vertexGroupCommiter fail (uv12) DAG dag2 = createDAG("testDAGVertexGroupCommitFail", true, false); dagClient = tezClient.submitDAG(dag2); dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.FAILED, dagClient.getDAGStatus(null).getState()); LOG.info(dagClient.getDAGStatus(null).getDiagnostics()); - Assert.assertTrue(StringUtils.join(dagClient.getDAGStatus(null).getDiagnostics(),",") + Assert.assertTrue(StringUtils.join(dagClient.getDAGStatus(null).getDiagnostics(), ",") .contains("fail output committer:uv12Out")); Assert.assertEquals(VertexStatus.State.SUCCEEDED, dagClient.getVertexStatus("v1", null).getState()); Assert.assertEquals(VertexStatus.State.SUCCEEDED, dagClient.getVertexStatus("v2", null).getState()); @@ -1046,7 +1049,7 @@ public void testCommitOutputOnDAGSuccess() throws Exception { dagClient.waitForCompletion(); LOG.info(dagClient.getDAGStatus(null).getDiagnostics()); Assert.assertEquals(DAGStatus.State.FAILED, dagClient.getDAGStatus(null).getState()); - Assert.assertTrue(StringUtils.join(dagClient.getDAGStatus(null).getDiagnostics(),",") + Assert.assertTrue(StringUtils.join(dagClient.getDAGStatus(null).getDiagnostics(), ",") .contains("fail output committer:v3Out")); Assert.assertEquals(VertexStatus.State.SUCCEEDED, dagClient.getVertexStatus("v1", null).getState()); Assert.assertEquals(VertexStatus.State.SUCCEEDED, dagClient.getVertexStatus("v2", null).getState()); @@ -1058,7 +1061,7 @@ public void testCommitOutputOnDAGSuccess() throws Exception { dagClient.waitForCompletion(); LOG.info(dagClient.getDAGStatus(null).getDiagnostics()); Assert.assertEquals(DAGStatus.State.FAILED, dagClient.getDAGStatus(null).getState()); - String diag = StringUtils.join(dagClient.getDAGStatus(null).getDiagnostics(),","); + String diag = StringUtils.join(dagClient.getDAGStatus(null).getDiagnostics(), ","); Assert.assertTrue(diag.contains("fail output committer:uv12Out") || diag.contains("fail output committer:v3Out")); Assert.assertEquals(VertexStatus.State.SUCCEEDED, dagClient.getVertexStatus("v1", null).getState()); @@ -1068,7 +1071,7 @@ public void testCommitOutputOnDAGSuccess() throws Exception { tezClient.stop(); } - @Test (timeout = 60000) + @Test(timeout = 60000) public void testCommitOutputOnVertexSuccess() throws Exception { TezConfiguration tezconf = new TezConfiguration(defaultConf); tezconf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false); @@ -1080,14 +1083,14 @@ public void testCommitOutputOnVertexSuccess() throws Exception { DAGClient dagClient = tezClient.submitDAG(dag1); dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState()); - + // vertexGroupCommiter fail (uv12) DAG dag2 = createDAG("testDAGVertexGroupCommitFail", true, false); dagClient = tezClient.submitDAG(dag2); dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.FAILED, dagClient.getDAGStatus(null).getState()); LOG.info(dagClient.getDAGStatus(null).getDiagnostics()); - Assert.assertTrue(StringUtils.join(dagClient.getDAGStatus(null).getDiagnostics(),",") + Assert.assertTrue(StringUtils.join(dagClient.getDAGStatus(null).getDiagnostics(), ",") .contains("fail output committer:uv12Out")); Assert.assertEquals(VertexStatus.State.SUCCEEDED, dagClient.getVertexStatus("v1", null).getState()); Assert.assertEquals(VertexStatus.State.SUCCEEDED, dagClient.getVertexStatus("v2", null).getState()); @@ -1106,14 +1109,14 @@ public void testCommitOutputOnVertexSuccess() throws Exception { dagClient.waitForCompletion(); LOG.info(dagClient.getDAGStatus(null).getDiagnostics()); Assert.assertEquals(DAGStatus.State.FAILED, dagClient.getDAGStatus(null).getState()); - Assert.assertTrue(StringUtils.join(dagClient.getDAGStatus(null).getDiagnostics(),",") + Assert.assertTrue(StringUtils.join(dagClient.getDAGStatus(null).getDiagnostics(), ",") .contains("Commit failed")); Assert.assertEquals(VertexStatus.State.SUCCEEDED, dagClient.getVertexStatus("v1", null).getState()); Assert.assertEquals(VertexStatus.State.SUCCEEDED, dagClient.getVertexStatus("v2", null).getState()); Assert.assertEquals(VertexStatus.State.FAILED, dagClient.getVertexStatus("v3", null).getState()); - Assert.assertTrue(StringUtils.join(dagClient.getVertexStatus("v3", null).getDiagnostics(),",") + Assert.assertTrue(StringUtils.join(dagClient.getVertexStatus("v3", null).getDiagnostics(), ",") .contains("fail output committer:v3Out")); - + // both committers fail DAG dag4 = createDAG("testDAGBothCommitsFail", true, true); dagClient = tezClient.submitDAG(dag4); @@ -1121,7 +1124,7 @@ public void testCommitOutputOnVertexSuccess() throws Exception { Assert.assertEquals(DAGStatus.State.FAILED, dagClient.getDAGStatus(null).getState()); LOG.info(dagClient.getDAGStatus(null).getDiagnostics()); Assert.assertEquals(DAGStatus.State.FAILED, dagClient.getDAGStatus(null).getState()); - String diag = StringUtils.join(dagClient.getDAGStatus(null).getDiagnostics(),","); + String diag = StringUtils.join(dagClient.getDAGStatus(null).getDiagnostics(), ","); Assert.assertTrue(diag.contains("fail output committer:uv12Out") || diag.contains("fail output committer:v3Out")); Assert.assertEquals(VertexStatus.State.SUCCEEDED, dagClient.getVertexStatus("v1", null).getState()); @@ -1131,7 +1134,7 @@ public void testCommitOutputOnVertexSuccess() throws Exception { // or killed ( uv12 commit fail before commit of v3) if (v3State.equals(VertexStatus.State.FAILED)) { LOG.info("v3 is failed"); - Assert.assertTrue(StringUtils.join(dagClient.getVertexStatus("v3", null).getDiagnostics(),",") + Assert.assertTrue(StringUtils.join(dagClient.getVertexStatus("v3", null).getDiagnostics(), ",") .contains("fail output committer:v3Out")); } else { Assert.assertEquals(VertexStatus.State.KILLED, v3State); @@ -1139,7 +1142,7 @@ public void testCommitOutputOnVertexSuccess() throws Exception { tezClient.stop(); } - + public static class FailingOutputCommitter extends OutputCommitter { boolean failOnCommit = false; @@ -1218,13 +1221,13 @@ public void testDAGFinishedRecoveryError() throws Exception { DAGClient dagClient = tezClient.submitDAG(dag); dagClient.waitForCompletion(); - while(!mockApp.getShutdownHandler().wasShutdownInvoked()) { + while (!mockApp.getShutdownHandler().wasShutdownInvoked()) { Thread.sleep(100); } Assert.assertEquals(DAGState.SUCCEEDED, mockApp.getContext().getCurrentDAG().getState()); Assert.assertEquals(DAGAppMasterState.FAILED, mockApp.getState()); - Assert.assertTrue(StringUtils.join(mockApp.getDiagnostics(),",") + Assert.assertTrue(StringUtils.join(mockApp.getDiagnostics(), ",") .contains("Recovery had a fatal error, shutting down session after" + - " DAG completion")); + " DAG completion")); } } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestPreemption.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestPreemption.java index 10917f0217..722c9f6070 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestPreemption.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestPreemption.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app; @@ -49,10 +49,10 @@ import org.junit.Test; public class TestPreemption { - + static Configuration defaultConf; static FileSystem localFs; - + static { try { defaultConf = new Configuration(false); @@ -65,26 +65,26 @@ public class TestPreemption { throw new RuntimeException("init failure", e); } } - - MockDAGAppMaster mockApp; + + MockDAGAppMaster mockApp; MockContainerLauncher mockLauncher; - + int dagCount = 0; - + DAG createDAG(DataMovementType dmType) { DAG dag = DAG.create("test-" + dagCount++); Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5); Vertex vB = Vertex.create("B", ProcessorDescriptor.create("Proc.class"), 5); - Edge eAB = Edge.create(vA, vB, - EdgeProperty.create(dmType, DataSourceType.PERSISTED, - SchedulingType.SEQUENTIAL, OutputDescriptor.create("O.class"), - InputDescriptor.create("I.class"))); - + Edge eAB = Edge.create(vA, vB, + EdgeProperty.create(dmType, DataSourceType.PERSISTED, + SchedulingType.SEQUENTIAL, OutputDescriptor.create("O.class"), + InputDescriptor.create("I.class"))); + dag.addVertex(vA).addVertex(vB).addEdge(eAB); return dag; } - - @Test (timeout = 5000) + + @Test(timeout = 5000) public void testPreemptionWithoutSession() throws Exception { System.out.println("TestPreemptionWithoutSession"); TezConfiguration tezconf = new TezConfiguration(defaultConf); @@ -93,7 +93,7 @@ public void testPreemptionWithoutSession() throws Exception { MockTezClient tezClient = new MockTezClient("testPreemption", tezconf, false, null, null, null, mockAppLauncherGoFlag, false, false, 2, 2); tezClient.start(); - + DAGClient dagClient = tezClient.submitDAG(createDAG(DataMovementType.SCATTER_GATHER)); // now the MockApp has been started. sync with it to get the launcher syncWithMockAppLauncher(false, mockAppLauncherGoFlag, tezClient); @@ -110,20 +110,20 @@ public void testPreemptionWithoutSession() throws Exception { mockLauncher.preemptContainerForTask(taId.getTaskID(), upToTaskVersion); mockLauncher.startScheduling(true); - + dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState()); - for (int i=0; i<=upToTaskVersion; ++i) { - TezTaskAttemptID testTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), i); + for (int i = 0; i <= upToTaskVersion; ++i) { + TezTaskAttemptID testTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), i); TaskAttemptImpl taImpl = dagImpl.getTaskAttempt(testTaId); Assert.assertEquals(TaskAttemptStateInternal.KILLED, taImpl.getInternalState()); } - + tezClient.stop(); } - - @Test (timeout = 30000) + + @Test(timeout = 30000) public void testPreemptionWithSession() throws Exception { System.out.println("TestPreemptionWithSession"); MockTezClient tezClient = createTezSession(); @@ -141,7 +141,7 @@ public void testPreemptionWithSession() throws Exception { testPreemptionMultiple(tezClient, createDAG(DataMovementType.ONE_TO_ONE), 1, "1-1"); tezClient.stop(); } - + MockTezClient createTezSession() throws Exception { TezConfiguration tezconf = new TezConfiguration(defaultConf); tezconf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 0); @@ -152,9 +152,9 @@ MockTezClient createTezSession() throws Exception { syncWithMockAppLauncher(false, mockAppLauncherGoFlag, tezClient); return tezClient; } - - void syncWithMockAppLauncher(boolean allowScheduling, AtomicBoolean mockAppLauncherGoFlag, - MockTezClient tezClient) throws Exception { + + void syncWithMockAppLauncher(boolean allowScheduling, AtomicBoolean mockAppLauncherGoFlag, + MockTezClient tezClient) throws Exception { synchronized (mockAppLauncherGoFlag) { while (!mockAppLauncherGoFlag.get()) { mockAppLauncherGoFlag.wait(); @@ -163,9 +163,9 @@ void syncWithMockAppLauncher(boolean allowScheduling, AtomicBoolean mockAppLaunc mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(allowScheduling); mockAppLauncherGoFlag.notify(); - } + } } - + void testPreemptionSingle(MockTezClient tezClient, DAG dag, int vertexIndex, String info) throws Exception { testPreemptionJob(tezClient, dag, vertexIndex, 0, info + "-Single"); @@ -177,31 +177,31 @@ void testPreemptionMultiple(MockTezClient tezClient, DAG dag, int vertexIndex, S } void testPreemptionJob(MockTezClient tezClient, DAG dag, int vertexIndex, - int upToTaskVersion, String info) throws Exception { + int upToTaskVersion, String info) throws Exception { System.out.println("TestPreemption - Running - " + info); TezConfiguration tezconf = new TezConfiguration(defaultConf); tezconf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 0); - + mockLauncher.startScheduling(false); // turn off scheduling to block DAG before submitting it DAGClient dagClient = tezClient.submitDAG(dag); - + DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG(); TezVertexID vertexId = TezVertexID.getInstance(dagImpl.getID(), vertexIndex); TezTaskAttemptID taId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0); mockLauncher.preemptContainerForTask(taId.getTaskID(), upToTaskVersion); mockLauncher.startScheduling(true); - + dagClient.waitForCompletion(); Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState()); - - for (int i=0; i<=upToTaskVersion; ++i) { - TezTaskAttemptID testTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), i); + + for (int i = 0; i <= upToTaskVersion; ++i) { + TezTaskAttemptID testTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), i); TaskAttemptImpl taImpl = dagImpl.getTaskAttempt(testTaId); Assert.assertEquals(TaskAttemptStateInternal.KILLED, taImpl.getInternalState()); Assert.assertEquals(TaskAttemptTerminationCause.EXTERNAL_PREEMPTION, taImpl.getTerminationCause()); } - + System.out.println("TestPreemption - Done running - " + info); } } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java index 57148ee7e9..17b3886e8f 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestRecoveryParser.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -164,7 +164,7 @@ public void testSkipAllOtherEvents_1() throws IOException { ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); TezDAGID dagID = TezDAGID.getInstance(appId, 1); AppContext appContext = mock(AppContext.class); - when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath+"/1")); + when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1")); when(appContext.getClock()).thenReturn(new SystemClock()); DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan(); @@ -180,11 +180,11 @@ public void testSkipAllOtherEvents_1() throws IOException { rService.handle(new DAGHistoryEvent(dagID, new DAGInitializedEvent(dagID, 1L, "user", dagPlan.getName(), null))); // only for testing, DAGCommitStartedEvent is not supposed to happen at this time. - rService.handle(new DAGHistoryEvent(dagID,new DAGCommitStartedEvent(dagID, System.currentTimeMillis()))); + rService.handle(new DAGHistoryEvent(dagID, new DAGCommitStartedEvent(dagID, System.currentTimeMillis()))); rService.stop(); // write data in attempt_2 - when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath+"/2")); + when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/2")); rService = new RecoveryService(appContext); rService.init(conf); rService.start(); @@ -204,13 +204,13 @@ public void testSkipAllOtherEvents_1() throws IOException { } // skipAllOtherEvents due to dag finished - @Test (timeout = 5000) + @Test(timeout = 5000) public void testSkipAllOtherEvents_2() throws IOException { ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); TezDAGID dagID = TezDAGID.getInstance(appId, 1); AppContext appContext = mock(AppContext.class); - when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath+"/1")); + when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1")); when(appContext.getClock()).thenReturn(new SystemClock()); DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan(); @@ -232,12 +232,12 @@ public void testSkipAllOtherEvents_2() throws IOException { rService.stop(); // write data in attempt_2 - when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath+"/2")); + when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/2")); rService = new RecoveryService(appContext); rService.init(conf); rService.start(); rService.handle(new DAGHistoryEvent(dagID, - new DAGStartedEvent(dagID, 1L, "user", "dag1"))); + new DAGStartedEvent(dagID, 1L, "user", "dag1"))); rService.stop(); DAGRecoveryData dagData = parser.parseRecoveryData(); @@ -257,7 +257,7 @@ public void testLastCorruptedRecoveryRecord() throws IOException { ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); TezDAGID dagID = TezDAGID.getInstance(appId, 1); AppContext appContext = mock(AppContext.class); - when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath+"/1")); + when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1")); when(appContext.getClock()).thenReturn(new SystemClock()); when(appContext.getHadoopShim()).thenReturn(new DefaultHadoopShim()); when(appContext.getApplicationID()).thenReturn(appId); @@ -278,7 +278,7 @@ public void testLastCorruptedRecoveryRecord() throws IOException { rService.stop(); // write data in attempt_2 - when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath+"/2")); + when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/2")); rService = new RecoveryService(appContext); rService.init(conf); rService.start(); @@ -303,7 +303,7 @@ public void testLastCorruptedSummaryRecord() throws IOException { ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); TezDAGID dagID = TezDAGID.getInstance(appId, 1); AppContext appContext = mock(AppContext.class); - when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath+"/1")); + when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1")); when(appContext.getClock()).thenReturn(new SystemClock()); // write data in attempt_1 @@ -332,12 +332,12 @@ public void testLastCorruptedSummaryRecord() throws IOException { } } - @Test(timeout=5000) + @Test(timeout = 5000) public void testRecoverableSummary_DAGInCommitting() throws IOException { ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); TezDAGID dagID = TezDAGID.getInstance(appId, 1); AppContext appContext = mock(AppContext.class); - when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath+"/1")); + when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1")); when(appContext.getClock()).thenReturn(new SystemClock()); when(mockDAGImpl.getID()).thenReturn(dagID); @@ -362,14 +362,14 @@ public void testRecoverableSummary_DAGInCommitting() throws IOException { assertTrue(dagData.nonRecoverable); assertTrue(dagData.reason.contains("DAG Commit was in progress")); } - - @Test(timeout=5000) + + @Test(timeout = 5000) public void testRecoverableSummary_DAGFinishCommitting() throws IOException { ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); TezDAGID dagID = TezDAGID.getInstance(appId, 1); AppContext appContext = mock(AppContext.class); - when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath+"/1")); + when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1")); when(appContext.getClock()).thenReturn(new SystemClock()); when(mockDAGImpl.getID()).thenReturn(dagID); @@ -400,12 +400,12 @@ public void testRecoverableSummary_DAGFinishCommitting() throws IOException { assertTrue(dagData.isCompleted); } - @Test(timeout=5000) + @Test(timeout = 5000) public void testRecoverableSummary_VertexInCommitting() throws IOException { ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); TezDAGID dagID = TezDAGID.getInstance(appId, 1); AppContext appContext = mock(AppContext.class); - when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath+"/1")); + when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1")); when(appContext.getClock()).thenReturn(new SystemClock()); when(mockDAGImpl.getID()).thenReturn(dagID); @@ -431,12 +431,12 @@ public void testRecoverableSummary_VertexInCommitting() throws IOException { assertTrue(dagData.reason.contains("Vertex Commit was in progress")); } - @Test(timeout=5000) + @Test(timeout = 5000) public void testRecoverableSummary_VertexFinishCommitting() throws IOException { ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); TezDAGID dagID = TezDAGID.getInstance(appId, 1); AppContext appContext = mock(AppContext.class); - when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath+"/1")); + when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1")); when(appContext.getClock()).thenReturn(new SystemClock()); when(mockDAGImpl.getID()).thenReturn(dagID); when(appContext.getHadoopShim()).thenReturn(new DefaultHadoopShim()); @@ -458,8 +458,8 @@ public void testRecoverableSummary_VertexFinishCommitting() throws IOException { rService.handle(new DAGHistoryEvent(dagID, new VertexCommitStartedEvent(vertexId, 0L))); rService.handle(new DAGHistoryEvent(dagID, - new VertexFinishedEvent(vertexId, "v1", 10, 0L, 0L, - 0L, 0L, 0L, VertexState.SUCCEEDED, + new VertexFinishedEvent(vertexId, "v1", 10, 0L, 0L, + 0L, 0L, 0L, VertexState.SUCCEEDED, "", null, null, null, null))); rService.stop(); @@ -468,12 +468,12 @@ public void testRecoverableSummary_VertexFinishCommitting() throws IOException { assertFalse(dagData.nonRecoverable); } - @Test(timeout=5000) + @Test(timeout = 5000) public void testRecoverableSummary_VertexGroupInCommitting() throws IOException { ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); TezDAGID dagID = TezDAGID.getInstance(appId, 1); AppContext appContext = mock(AppContext.class); - when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath+"/1")); + when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1")); when(appContext.getClock()).thenReturn(new SystemClock()); when(mockDAGImpl.getID()).thenReturn(dagID); when(appContext.getHadoopShim()).thenReturn(new DefaultHadoopShim()); @@ -492,7 +492,7 @@ public void testRecoverableSummary_VertexGroupInCommitting() throws IOException null, "user", new Configuration(), null, null))); // It should be fine to skip other events, just for testing. rService.handle(new DAGHistoryEvent(dagID, - new VertexGroupCommitStartedEvent(dagID, "group_1", + new VertexGroupCommitStartedEvent(dagID, "group_1", Lists.newArrayList(TezVertexID.getInstance(dagID, 0), TezVertexID.getInstance(dagID, 1)), 0L))); rService.stop(); @@ -501,13 +501,13 @@ public void testRecoverableSummary_VertexGroupInCommitting() throws IOException assertTrue(dagData.nonRecoverable); assertTrue(dagData.reason.contains("Vertex Group Commit was in progress")); } - - @Test(timeout=5000) + + @Test(timeout = 5000) public void testRecoverableSummary_VertexGroupFinishCommitting() throws IOException { ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); TezDAGID dagID = TezDAGID.getInstance(appId, 1); AppContext appContext = mock(AppContext.class); - when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath+"/1")); + when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1")); when(appContext.getClock()).thenReturn(new SystemClock()); when(mockDAGImpl.getID()).thenReturn(dagID); when(appContext.getHadoopShim()).thenReturn(new DefaultHadoopShim()); @@ -528,34 +528,34 @@ public void testRecoverableSummary_VertexGroupFinishCommitting() throws IOExcept TezVertexID v0 = TezVertexID.getInstance(dagID, 0); TezVertexID v1 = TezVertexID.getInstance(dagID, 1); rService.handle(new DAGHistoryEvent(dagID, - new VertexGroupCommitStartedEvent(dagID, "group_1", + new VertexGroupCommitStartedEvent(dagID, "group_1", Lists.newArrayList(v0, v1), 0L))); rService.handle(new DAGHistoryEvent(dagID, - new VertexGroupCommitFinishedEvent(dagID, "group_1", + new VertexGroupCommitFinishedEvent(dagID, "group_1", Lists.newArrayList(v0, v1), 0L))); // also write VertexFinishedEvent, otherwise it is still non-recoverable // when checking with non-summary event rService.handle(new DAGHistoryEvent(dagID, - new VertexFinishedEvent(v0, "v1", 10, 0L, 0L, - 0L, 0L, 0L, VertexState.SUCCEEDED, + new VertexFinishedEvent(v0, "v1", 10, 0L, 0L, + 0L, 0L, 0L, VertexState.SUCCEEDED, "", null, null, null, null))); rService.handle(new DAGHistoryEvent(dagID, - new VertexFinishedEvent(v1, "v1", 10, 0L, 0L, - 0L, 0L, 0L, VertexState.SUCCEEDED, + new VertexFinishedEvent(v1, "v1", 10, 0L, 0L, + 0L, 0L, 0L, VertexState.SUCCEEDED, "", null, null, null, null))); rService.stop(); - + DAGRecoveryData dagData = parser.parseRecoveryData(); assertEquals(dagID, dagData.recoveredDagID); assertFalse(dagData.nonRecoverable); } - - @Test(timeout=5000) + + @Test(timeout = 5000) public void testRecoverableNonSummary1() throws IOException { ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); TezDAGID dagID = TezDAGID.getInstance(appId, 1); AppContext appContext = mock(AppContext.class); - when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath+"/1")); + when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1")); when(appContext.getClock()).thenReturn(new SystemClock()); when(mockDAGImpl.getID()).thenReturn(dagID); when(appContext.getHadoopShim()).thenReturn(new DefaultHadoopShim()); @@ -578,8 +578,8 @@ public void testRecoverableNonSummary1() throws IOException { rService.handle(new DAGHistoryEvent(dagID, new VertexCommitStartedEvent(vertexId, 0L))); rService.handle(new DAGHistoryEvent(dagID, - new VertexFinishedEvent(vertexId, "v1", 10, 0L, 0L, - 0L, 0L, 0L, VertexState.SUCCEEDED, + new VertexFinishedEvent(vertexId, "v1", 10, 0L, 0L, + 0L, 0L, 0L, VertexState.SUCCEEDED, "", null, null, null, null))); rService.stop(); @@ -587,13 +587,13 @@ public void testRecoverableNonSummary1() throws IOException { assertTrue(dagData.nonRecoverable); assertTrue(dagData.reason.contains("Vertex has been committed, but its full recovery events are not seen")); } - - @Test(timeout=5000) + + @Test(timeout = 5000) public void testRecoverableNonSummary2() throws IOException { ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); TezDAGID dagID = TezDAGID.getInstance(appId, 1); AppContext appContext = mock(AppContext.class); - when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath+"/1")); + when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1")); when(appContext.getClock()).thenReturn(new SystemClock()); when(mockDAGImpl.getID()).thenReturn(dagID); @@ -612,25 +612,25 @@ public void testRecoverableNonSummary2() throws IOException { // It should be fine to skip other events, just for testing. TezVertexID vertexId = TezVertexID.getInstance(dagID, 0); rService.handle(new DAGHistoryEvent(dagID, - new VertexGroupCommitStartedEvent(dagID, "group_1", + new VertexGroupCommitStartedEvent(dagID, "group_1", Lists.newArrayList(TezVertexID.getInstance(dagID, 0), TezVertexID.getInstance(dagID, 1)), 0L))); rService.handle(new DAGHistoryEvent(dagID, - new VertexGroupCommitFinishedEvent(dagID, "group_1", + new VertexGroupCommitFinishedEvent(dagID, "group_1", Lists.newArrayList(TezVertexID.getInstance(dagID, 0), TezVertexID.getInstance(dagID, 1)), 0L))); rService.stop(); DAGRecoveryData dagData = parser.parseRecoveryData(); assertTrue(dagData.nonRecoverable); assertTrue(dagData.reason.contains("Vertex has been committed as member of vertex group" - + ", but its full recovery events are not seen")); + + ", but its full recovery events are not seen")); } - @Test(timeout=20000) + @Test(timeout = 20000) public void testRecoveryLargeEventData() throws IOException { ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); TezDAGID dagID = TezDAGID.getInstance(appId, 1); AppContext appContext = mock(AppContext.class); - when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath+"/1")); + when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1")); when(appContext.getClock()).thenReturn(new SystemClock()); when(mockDAGImpl.getID()).thenReturn(dagID); when(appContext.getHadoopShim()).thenReturn(new DefaultHadoopShim()); @@ -680,7 +680,7 @@ public void testRecoveryLargeEventData() throws IOException { VertexConfigurationDoneEvent vertexConfigurationDoneEvent = new VertexConfigurationDoneEvent( v0Id, 0, 100000, vertexLocationHint, null, null, false); // Verify large protobuf message - assertTrue(vertexConfigurationDoneEvent.toProto().getSerializedSize() > PROTOBUF_DEFAULT_SIZE_LIMIT ); + assertTrue(vertexConfigurationDoneEvent.toProto().getSerializedSize() > PROTOBUF_DEFAULT_SIZE_LIMIT); rService.handle(new DAGHistoryEvent(dagID, vertexConfigurationDoneEvent)); rService.stop(); @@ -694,13 +694,13 @@ public void testRecoveryLargeEventData() throws IOException { assertEquals(parsedVertexLocationHint.getTaskLocationHints().size(), 100000); } - @Test(timeout=5000) + @Test(timeout = 5000) public void testRecoveryData() throws IOException { ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); TezDAGID dagID = TezDAGID.getInstance(appId, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); AppContext appContext = mock(AppContext.class); - when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath+"/1")); + when(appContext.getCurrentRecoveryDir()).thenReturn(new Path(recoveryPath + "/1")); when(appContext.getClock()).thenReturn(new SystemClock()); when(mockDAGImpl.getID()).thenReturn(dagID); when(appContext.getHadoopShim()).thenReturn(new DefaultHadoopShim()); @@ -717,23 +717,23 @@ public void testRecoveryData() throws IOException { rService.handle(new DAGHistoryEvent(dagID, new DAGSubmittedEvent(dagID, 1L, dagPlan, ApplicationAttemptId.newInstance(appId, 1), null, "user", new Configuration(), null, null))); - DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagID, 100L, + DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagID, 100L, "user", "dagName", null); DAGStartedEvent dagStartedEvent = new DAGStartedEvent(dagID, 0L, "user", "dagName"); rService.handle(new DAGHistoryEvent(dagID, dagInitedEvent)); rService.handle(new DAGHistoryEvent(dagID, dagStartedEvent)); - + // 3 vertices of this dag: v0, v1, v2 TezVertexID v0Id = TezVertexID.getInstance(dagID, 0); TezVertexID v1Id = TezVertexID.getInstance(dagID, 1); TezVertexID v2Id = TezVertexID.getInstance(dagID, 2); // v0 VertexInitializedEvent - VertexInitializedEvent v0InitedEvent = new VertexInitializedEvent( + VertexInitializedEvent v0InitedEvent = new VertexInitializedEvent( v0Id, "v0", 200L, 400L, 2, null, null, null, null); rService.handle(new DAGHistoryEvent(dagID, v0InitedEvent)); // v1 VertexFinishedEvent(KILLED) - VertexFinishedEvent v1FinishedEvent = new VertexFinishedEvent(v1Id, "v1", 2, 300L, 400L, - 500L, 600L, 700L, VertexState.KILLED, + VertexFinishedEvent v1FinishedEvent = new VertexFinishedEvent(v1Id, "v1", 2, 300L, 400L, + 500L, 600L, 700L, VertexState.KILLED, "", null, null, null, null); rService.handle(new DAGHistoryEvent(dagID, v1FinishedEvent)); // v2 VertexInitializedEvent -> VertexStartedEvent @@ -768,17 +768,17 @@ public void testRecoveryData() throws IOException { NodeId nodeId = NodeId.newInstance("localhost", 9999); TezTaskAttemptID ta0t0v2Id = TezTaskAttemptID.getInstance(t0v2Id, 0); TaskAttemptStartedEvent ta0t0v2StartedEvent = new TaskAttemptStartedEvent( - ta0t0v2Id, "v1", 0L, containerId, + ta0t0v2Id, "v1", 0L, containerId, nodeId, "", "", ""); rService.handle(new DAGHistoryEvent(dagID, ta0t0v2StartedEvent)); // attempts under t2v2 TezTaskAttemptID ta0t2v2Id = TezTaskAttemptID.getInstance(t2v2Id, 0); TaskAttemptStartedEvent ta0t2v2StartedEvent = new TaskAttemptStartedEvent( - ta0t2v2Id, "v1", 500L, containerId, + ta0t2v2Id, "v1", 500L, containerId, nodeId, "", "", ""); rService.handle(new DAGHistoryEvent(dagID, ta0t2v2StartedEvent)); TaskAttemptFinishedEvent ta0t2v2FinishedEvent = new TaskAttemptFinishedEvent( - ta0t2v2Id, "v1", 500L, 600L, + ta0t2v2Id, "v1", 500L, 600L, TaskAttemptState.SUCCEEDED, null, null, "", null, null, null, 0L, null, 0L, null, null, null, null, null); rService.handle(new DAGHistoryEvent(dagID, ta0t2v2FinishedEvent)); @@ -830,7 +830,7 @@ public void testRecoveryData() throws IOException { // Simulate the behavior that summary event is written // but non-summary is not written to hdfs - public static class MockRecoveryService extends RecoveryService{ + public static class MockRecoveryService extends RecoveryService { public MockRecoveryService(AppContext appContext) { super(appContext); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestSpeculation.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestSpeculation.java index 302281af9f..bff0845295 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestSpeculation.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestSpeculation.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app; @@ -28,6 +28,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -112,7 +113,8 @@ public class TestSpeculation { * The interface Retry. */ @Retention(RetentionPolicy.RUNTIME) - public @interface Retry {} + public @interface Retry { + } /** * The type Retry rule. @@ -133,7 +135,7 @@ class RetryRule implements TestRule { @Override public Statement apply(final Statement base, - final Description description) { + final Description description) { return new Statement() { @Override public void evaluate() throws Throwable { @@ -240,7 +242,7 @@ public static Collection getTestParameters() { * * @param estimatorKlass the estimator klass */ - public TestSpeculation(Class estimatorKlass) { + public TestSpeculation(Class estimatorKlass) { this.estimatorClass = estimatorKlass; } @@ -269,7 +271,7 @@ MockTezClient createTezSession() throws Exception { * @throws Exception the exception */ void syncWithMockAppLauncher(boolean allowScheduling, AtomicBoolean mockAppLauncherGoFlag, - MockTezClient tezClient) throws Exception { + MockTezClient tezClient) throws Exception { synchronized (mockAppLauncherGoFlag) { while (!mockAppLauncherGoFlag.get()) { mockAppLauncherGoFlag.wait(); @@ -278,7 +280,7 @@ void syncWithMockAppLauncher(boolean allowScheduling, AtomicBoolean mockAppLaunc mockLauncher = mockApp.getContainerLauncher(); mockLauncher.startScheduling(allowScheduling); mockAppLauncherGoFlag.notify(); - } + } } /** @@ -287,7 +289,7 @@ void syncWithMockAppLauncher(boolean allowScheduling, AtomicBoolean mockAppLaunc * @throws Exception the exception */ @Retry - @Test (timeout = 30000) + @Test(timeout = 30000) public void testSingleTaskSpeculation() throws Exception { // Map Map confToExpected = new HashMap(); @@ -295,15 +297,15 @@ public void testSingleTaskSpeculation() throws Exception { confToExpected.put(100L, 2); confToExpected.put(-1L, 1); // Don't speculate defaultConf.setLong(TezConfiguration.TEZ_AM_SOONEST_RETRY_AFTER_NO_SPECULATE, 50); - for(Map.Entry entry : confToExpected.entrySet()) { + for (Map.Entry entry : confToExpected.entrySet()) { defaultConf.setLong( - TezConfiguration.TEZ_AM_LEGACY_SPECULATIVE_SINGLE_TASK_VERTEX_TIMEOUT, - entry.getKey()); + TezConfiguration.TEZ_AM_LEGACY_SPECULATIVE_SINGLE_TASK_VERTEX_TIMEOUT, + entry.getKey()); DAG dag = DAG.create("test"); Vertex vA = Vertex.create("A", - ProcessorDescriptor.create("Proc.class"), - 1); + ProcessorDescriptor.create("Proc.class"), + 1); dag.addVertex(vA); MockTezClient tezClient = createTezSession(); @@ -330,7 +332,7 @@ public void testSingleTaskSpeculation() throws Exception { TaskAttempt killedAttempt = task.getAttempt(killedTaId); Joiner.on(",").join(killedAttempt.getDiagnostics()).contains("Killed as speculative attempt"); Assert.assertEquals(TaskAttemptTerminationCause.TERMINATED_EFFECTIVE_SPECULATION, - killedAttempt.getTerminationCause()); + killedAttempt.getTerminationCause()); } tezClient.stop(); } @@ -372,7 +374,7 @@ public void testBasicSpeculation(boolean withProgress) throws Exception { Assert.assertEquals(successTaId, task.getSuccessfulAttempt().getTaskAttemptID()); TaskAttempt killedAttempt = task.getAttempt(killedTaId); Joiner.on(",").join(killedAttempt.getDiagnostics()).contains("Killed as speculative attempt"); - Assert.assertEquals(TaskAttemptTerminationCause.TERMINATED_EFFECTIVE_SPECULATION, + Assert.assertEquals(TaskAttemptTerminationCause.TERMINATED_EFFECTIVE_SPECULATION, killedAttempt.getTerminationCause()); if (withProgress) { // without progress updates occasionally more than 1 task speculates @@ -386,7 +388,7 @@ public void testBasicSpeculation(boolean withProgress) throws Exception { } LegacySpeculator speculator = - (LegacySpeculator)(dagImpl.getVertex(vA.getName())).getSpeculator(); + (LegacySpeculator) (dagImpl.getVertex(vA.getName())).getSpeculator(); Assert.assertEquals(20, speculator.getMinimumAllowedSpeculativeTasks()); Assert.assertEquals(.2, speculator.getProportionTotalTasksSpeculatable(), 0); Assert.assertEquals(.25, speculator.getProportionRunningTasksSpeculatable(), 0); @@ -402,7 +404,7 @@ public void testBasicSpeculation(boolean withProgress) throws Exception { * @throws Exception the exception */ @Retry - @Test (timeout=30000) + @Test(timeout = 30000) public void testBasicSpeculationWithProgress() throws Exception { testBasicSpeculation(true); } @@ -413,7 +415,7 @@ public void testBasicSpeculationWithProgress() throws Exception { * @throws Exception the exception */ @Retry - @Test (timeout=30000) + @Test(timeout = 30000) public void testBasicSpeculationWithoutProgress() throws Exception { testBasicSpeculation(false); } @@ -424,7 +426,7 @@ public void testBasicSpeculationWithoutProgress() throws Exception { * @throws Exception the exception */ @Retry - @Test (timeout=30000) + @Test(timeout = 30000) public void testBasicSpeculationPerVertexConf() throws Exception { DAG dag = DAG.create("test"); String vNameNoSpec = "A"; @@ -485,14 +487,14 @@ public void testBasicSpeculationPerVertexConf() throws Exception { * @throws Exception the exception */ @Retry - @Test (timeout=30000) + @Test(timeout = 30000) public void testBasicSpeculationNotUseful() throws Exception { DAG dag = DAG.create("test"); Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5); dag.addVertex(vA); MockTezClient tezClient = createTezSession(); - + DAGClient dagClient = tezClient.submitDAG(dag); DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG(); TezVertexID vertexId = TezVertexID.getInstance(dagImpl.getID(), 0); @@ -511,7 +513,7 @@ public void testBasicSpeculationNotUseful() throws Exception { Assert.assertEquals(successTaId, task.getSuccessfulAttempt().getTaskAttemptID()); TaskAttempt killedAttempt = task.getAttempt(killedTaId); Joiner.on(",").join(killedAttempt.getDiagnostics()).contains("Killed speculative attempt as"); - Assert.assertEquals(TaskAttemptTerminationCause.TERMINATED_INEFFECTIVE_SPECULATION, + Assert.assertEquals(TaskAttemptTerminationCause.TERMINATED_INEFFECTIVE_SPECULATION, killedAttempt.getTerminationCause()); Assert.assertEquals(1, task.getCounters().findCounter(TaskCounter.NUM_SPECULATIONS) .getValue()); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager.java index 3935789860..242d24c8e5 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager.java @@ -99,8 +99,6 @@ public void testNoTaskCommSpecified() throws IOException, TezException { } catch (IllegalArgumentException e) { } - - } @Test(timeout = 5000) @@ -132,7 +130,6 @@ public void testCustomTaskCommSpecified() throws IOException, TezException { assertEquals(customTaskCommName, tcm.getTaskCommName(0)); assertEquals(bb, tcm.getTaskCommContext(0).getInitialUserPayload().getPayload()); - } finally { tcm.stop(); } @@ -222,7 +219,6 @@ public void testEventRouting() throws Exception { verify(tcm.getTestTaskComm(1)).initialize(); verify(tcm.getTestTaskComm(1)).start(); - ContainerId containerId1 = mock(ContainerId.class); tcm.registerRunningContainer(containerId1, 0); verify(tcm.getTestTaskComm(0)).registerRunningContainer(eq(containerId1), eq("host1"), @@ -232,7 +228,6 @@ public void testEventRouting() throws Exception { tcm.registerRunningContainer(containerId2, 1); verify(tcm.getTestTaskComm(1)).registerRunningContainer(eq(containerId2), eq("host1"), eq(3131)); - } finally { tcm.stop(); verify(tcm.getTaskCommunicator(0).getTaskCommunicator()).shutdown(); @@ -260,7 +255,6 @@ public void testReportFailureFromTaskCommunicator() throws TezException { List list = new LinkedList<>(); list.add(namedEntityDescriptor); - TaskCommunicatorManager taskCommManager = new TaskCommunicatorManager(appContext, mock(TaskHeartbeatHandler.class), mock(ContainerHeartbeatHandler.class), list); @@ -280,7 +274,6 @@ public void testReportFailureFromTaskCommunicator() throws TezException { .contains(ServicePluginErrorDefaults.SERVICE_UNAVAILABLE.name())); assertTrue(killEvent.getDiagnosticInfo().contains("[0:testTaskCommunicator]")); - reset(eventHandler); taskCommManager.dagComplete(dag); @@ -298,11 +291,9 @@ public void testReportFailureFromTaskCommunicator() throws TezException { assertTrue( event.getDiagnosticInfo().contains(ServicePluginErrorDefaults.INCONSISTENT_STATE.name())); assertTrue(event.getDiagnosticInfo().contains("[0:testTaskCommunicator]")); - } finally { taskCommManager.stop(); } - } @SuppressWarnings("unchecked") @@ -347,7 +338,6 @@ public void testTaskCommunicatorUserError() { assertTrue(event.getDiagnosticInfo().contains("DAG completion")); assertTrue(event.getDiagnosticInfo().contains(expectedId)); - when(appContext.getAllContainers().get(any()).getContainer().getNodeId()) .thenReturn(mock(NodeId.class)); @@ -364,12 +354,9 @@ public void testTaskCommunicatorUserError() { event.getError().getMessage().contains("TestException_" + "registerRunningContainer")); assertTrue(event.getDiagnosticInfo().contains("registering running Container")); assertTrue(event.getDiagnosticInfo().contains(expectedId)); - - } finally { taskCommunicatorManager.stop(); } - } private static class ExceptionAnswer implements Answer { @@ -403,7 +390,6 @@ static class TaskCommManagerForMultipleCommTest extends TaskCommunicatorManager private static final List taskCommNames = new LinkedList<>(); private static final List testTaskComms = new LinkedList<>(); - public static void reset() { numTaskComms.set(0); taskCommIndices.clear(); @@ -536,6 +522,7 @@ public Object getMetaInfo() { private static final String DAG_NAME = "dagName"; private static final int DAG_INDEX = 1; + public static class TaskCommForFailureTest extends TaskCommunicator { public TaskCommForFailureTest( diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager1.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager1.java index d7e62ee530..7e7b01cd50 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager1.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager1.java @@ -1,16 +1,16 @@ /* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app; @@ -130,14 +130,14 @@ public void setUp() throws TezException { eventHandler = mock(EventHandler.class); MockClock clock = new MockClock(); - + appContext = mock(AppContext.class); doReturn(eventHandler).when(appContext).getEventHandler(); doReturn(dag).when(appContext).getCurrentDAG(); doReturn(appAcls).when(appContext).getApplicationACLs(); doReturn(amContainerMap).when(appContext).getAllContainers(); doReturn(clock).when(appContext).getClock(); - + doReturn(appAttemptId).when(appContext).getApplicationAttemptId(); doReturn(credentials).when(appContext).getAppCredentials(); NodeId nodeId = NodeId.newInstance("localhost", 0); @@ -239,15 +239,15 @@ public void testGetTaskMultiplePulls() throws IOException { assertNull(containerTask); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testTaskEventRouting() throws Exception { - List events = Arrays.asList( - new TezEvent(new TaskStatusUpdateEvent(null, 0.0f, null, false), new EventMetaData(EventProducerConsumerType.PROCESSOR, - "v1", "v2", taskAttemptID)), - new TezEvent(DataMovementEvent.create(0, ByteBuffer.wrap(new byte[0])), new EventMetaData(EventProducerConsumerType.OUTPUT, - "v1", "v2", taskAttemptID)), - new TezEvent(new TaskAttemptCompletedEvent(), new EventMetaData(EventProducerConsumerType.SYSTEM, - "v1", "v2", taskAttemptID)) + List events = Arrays.asList( + new TezEvent(new TaskStatusUpdateEvent(null, 0.0f, null, false), new EventMetaData(EventProducerConsumerType.PROCESSOR, + "v1", "v2", taskAttemptID)), + new TezEvent(DataMovementEvent.create(0, ByteBuffer.wrap(new byte[0])), new EventMetaData(EventProducerConsumerType.OUTPUT, + "v1", "v2", taskAttemptID)), + new TezEvent(new TaskAttemptCompletedEvent(), new EventMetaData(EventProducerConsumerType.SYSTEM, + "v1", "v2", taskAttemptID)) ); generateHeartbeat(events, 0, 1, 0, new ArrayList()); @@ -259,29 +259,29 @@ public void testTaskEventRouting() throws Exception { final Event statusUpdateEvent = argAllValues.get(0); assertEquals("First event should be status update", TaskAttemptEventType.TA_STATUS_UPDATE, statusUpdateEvent.getType()); - assertEquals(false, ((TaskAttemptEventStatusUpdate)statusUpdateEvent).getReadErrorReported()); + assertEquals(false, ((TaskAttemptEventStatusUpdate) statusUpdateEvent).getReadErrorReported()); - final TaskAttemptEventTezEventUpdate taEvent = (TaskAttemptEventTezEventUpdate)argAllValues.get(1); + final TaskAttemptEventTezEventUpdate taEvent = (TaskAttemptEventTezEventUpdate) argAllValues.get(1); assertEquals(1, taEvent.getTezEvents().size()); assertEquals(EventType.DATA_MOVEMENT_EVENT, taEvent.getTezEvents().get(0).getEventType()); - - final TaskAttemptEvent taCompleteEvent = (TaskAttemptEvent)argAllValues.get(2); + + final TaskAttemptEvent taCompleteEvent = (TaskAttemptEvent) argAllValues.get(2); assertEquals(TaskAttemptEventType.TA_DONE, taCompleteEvent.getType()); - final VertexEventRouteEvent vertexRouteEvent = (VertexEventRouteEvent)argAllValues.get(3); + final VertexEventRouteEvent vertexRouteEvent = (VertexEventRouteEvent) argAllValues.get(3); assertEquals(1, vertexRouteEvent.getEvents().size()); assertEquals(EventType.DATA_MOVEMENT_EVENT, vertexRouteEvent.getEvents().get(0).getEventType()); } - - @Test (timeout = 5000) + + @Test(timeout = 5000) public void testTaskEventRoutingWithReadError() throws Exception { - List events = Arrays.asList( - new TezEvent(new TaskStatusUpdateEvent(null, 0.0f, null, false), null), - new TezEvent(InputReadErrorEvent.create("", 0, 0), new EventMetaData(EventProducerConsumerType.INPUT, - "v2", "v1", taskAttemptID)), - new TezEvent(new TaskAttemptCompletedEvent(), new EventMetaData(EventProducerConsumerType.SYSTEM, - "v1", "v2", taskAttemptID)) + List events = Arrays.asList( + new TezEvent(new TaskStatusUpdateEvent(null, 0.0f, null, false), null), + new TezEvent(InputReadErrorEvent.create("", 0, 0), new EventMetaData(EventProducerConsumerType.INPUT, + "v2", "v1", taskAttemptID)), + new TezEvent(new TaskAttemptCompletedEvent(), new EventMetaData(EventProducerConsumerType.SYSTEM, + "v1", "v2", taskAttemptID)) ); generateHeartbeat(events, 0, 1, 0, new ArrayList()); @@ -293,26 +293,25 @@ public void testTaskEventRoutingWithReadError() throws Exception { final Event statusUpdateEvent = argAllValues.get(0); assertEquals("First event should be status update", TaskAttemptEventType.TA_STATUS_UPDATE, statusUpdateEvent.getType()); - assertEquals(true, ((TaskAttemptEventStatusUpdate)statusUpdateEvent).getReadErrorReported()); + assertEquals(true, ((TaskAttemptEventStatusUpdate) statusUpdateEvent).getReadErrorReported()); final Event taFinishedEvent = argAllValues.get(1); assertEquals("Second event should be TA_DONE", TaskAttemptEventType.TA_DONE, taFinishedEvent.getType()); final Event vertexEvent = argAllValues.get(2); - final VertexEventRouteEvent vertexRouteEvent = (VertexEventRouteEvent)vertexEvent; + final VertexEventRouteEvent vertexRouteEvent = (VertexEventRouteEvent) vertexEvent; assertEquals("Third event should be routed to vertex", VertexEventType.V_ROUTE_EVENT, vertexEvent.getType()); assertEquals(EventType.INPUT_READ_ERROR_EVENT, vertexRouteEvent.getEvents().get(0).getEventType()); } - - @Test (timeout = 5000) + @Test(timeout = 5000) public void testTaskEventRoutingTaskAttemptOnly() throws Exception { List events = Arrays.asList( - new TezEvent(new TaskAttemptCompletedEvent(), new EventMetaData(EventProducerConsumerType.SYSTEM, - "v1", "v2", taskAttemptID)) + new TezEvent(new TaskAttemptCompletedEvent(), new EventMetaData(EventProducerConsumerType.SYSTEM, + "v1", "v2", taskAttemptID)) ); generateHeartbeat(events, 0, 1, 0, new ArrayList()); @@ -325,19 +324,19 @@ public void testTaskEventRoutingTaskAttemptOnly() throws Exception { assertEquals("only event should be route event", TaskAttemptEventType.TA_DONE, event.getType()); } - - @Test (timeout = 5000) + + @Test(timeout = 5000) public void testTaskHeartbeatResponse() throws Exception { List events = new ArrayList(); List eventsToSend = new ArrayList(); TaskHeartbeatResponse response = generateHeartbeat(events, 0, 1, 2, eventsToSend); - + assertEquals(2, response.getNextFromEventId()); assertEquals(eventsToSend, response.getEvents()); } //try 10 times to allocate random port, fail it if no one is succeed. - @Test (timeout = 5000) + @Test(timeout = 5000) public void testPortRange() { boolean succeedToAllocate = false; Random rand = new Random(); @@ -354,7 +353,7 @@ public void testPortRange() { } // TODO TEZ-2003 Move this into TestTezTaskCommunicator. Potentially other tests as well. - @Test (timeout= 5000) + @Test(timeout = 5000) public void testPortRange_NotSpecified() throws IOException, TezException { Configuration conf = new Configuration(); JobTokenIdentifier identifier = new JobTokenIdentifier(new Text( @@ -377,7 +376,7 @@ private boolean testPortRange(int port) { boolean succeedToAllocate = true; try { Configuration conf = new Configuration(); - + JobTokenIdentifier identifier = new JobTokenIdentifier(new Text( "fakeIdentifier")); Token sessionToken = new Token(identifier, @@ -412,8 +411,8 @@ private boolean testPortRange(int port) { } private TaskHeartbeatResponse generateHeartbeat(List events, - int fromEventId, int maxEvents, int nextFromEventId, - List sendEvents) throws IOException, TezException { + int fromEventId, int maxEvents, int nextFromEventId, + List sendEvents) throws IOException, TezException { ContainerId containerId = createContainerId(appId, 1); Vertex vertex = mock(Vertex.class); @@ -455,7 +454,6 @@ public TaskCommunicatorManagerInterfaceImplForTest(AppContext context, TaskCommunicator createDefaultTaskCommunicator(TaskCommunicatorContext taskCommunicatorContext) { return new TezTaskCommunicatorImplForTest(taskCommunicatorContext); } - } private static class TezTaskCommunicatorImplForTest extends TezTaskCommunicatorImpl { diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager2.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager2.java index ac60001a46..cdcf858999 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager2.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorManager2.java @@ -204,7 +204,6 @@ public void testTaskAttemptFailureViaContext() throws IOException, TezException wrapper.registerRunningContainer(containerId2); wrapper.registerTaskAttempt(containerId2, amContainerTask2); - // non-fatal wrapper.getTaskCommunicatorManager() .taskFailed(taskSpec1.getTaskAttemptID(), TaskFailureType.NON_FATAL, @@ -278,7 +277,6 @@ private static class TaskCommunicatorManagerWrapperForTest { TezConstants.getTezYarnServicePluginName(), null).setUserPayload(userPayload))); } - TaskCommunicatorManager getTaskCommunicatorManager() { return taskCommunicatorManager; } @@ -303,13 +301,11 @@ private TaskSpec createTaskSpec() { return taskSpec; } - @SuppressWarnings("deprecation") private ContainerId createContainerId(int id) { ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); ContainerId containerId = ContainerId.newInstance(appAttemptId, id); return containerId; } - } } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorWrapper.java b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorWrapper.java index e89cc990b1..6248f88247 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorWrapper.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/TestTaskCommunicatorWrapper.java @@ -39,5 +39,4 @@ public void testDelegation() throws Exception { PluginWrapperTestHelpers.testDelegation(TaskCommunicatorWrapper.class, TaskCommunicator.class, Sets.newHashSet("getTaskCommunicator")); } - } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/TestRootInputInitializerManager.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/TestRootInputInitializerManager.java index 2c5aa337b7..676b386667 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/TestRootInputInitializerManager.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/TestRootInputInitializerManager.java @@ -90,7 +90,6 @@ public void testEventBeforeSuccess() throws Exception { String srcVertexName = "srcVertexName"; List eventList = Lists.newLinkedList(); - // First Attempt send event TezTaskAttemptID srcTaskAttemptId11 = TezTaskAttemptID.getInstance(srcTaskId1, 1); EventMetaData sourceInfo11 = @@ -165,7 +164,6 @@ public void testSuccessBeforeEvent() throws Exception { String srcVertexName = "srcVertexName"; List eventList = Lists.newLinkedList(); - // First Attempt send event TezTaskAttemptID srcTaskAttemptId11 = TezTaskAttemptID.getInstance(srcTaskId1, 1); EventMetaData sourceInfo11 = @@ -188,7 +186,6 @@ public void testSuccessBeforeEvent() throws Exception { reset(initializer); - TezTaskAttemptID srcTaskAttemptId12 = TezTaskAttemptID.getInstance(srcTaskId1, 2); // 2nd attempt succeeded initializerWrapper.onTaskSucceeded(srcVertexName, srcTaskId1, srcTaskAttemptId12.getId()); @@ -206,8 +203,7 @@ public void testSuccessBeforeEvent() throws Exception { verify(initializer, never()).handleInputInitializerEvent(any()); } - - @Test (timeout = 5000) + @Test(timeout = 5000) public void testCorrectUgiUsage() throws TezException, InterruptedException { Vertex vertex = mock(Vertex.class); doReturn(mock(TezVertexID.class)).when(vertex).getVertexId(); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/TestStateChangeNotifier.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/TestStateChangeNotifier.java index b43782e165..1059bb4c1f 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/TestStateChangeNotifier.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/TestStateChangeNotifier.java @@ -47,7 +47,7 @@ import org.slf4j.LoggerFactory; public class TestStateChangeNotifier { - + // uses the thread based notification code path but effectively blocks update // events till listeners have been notified public static class StateChangeNotifierForTest extends StateChangeNotifier { @@ -63,14 +63,14 @@ public void reset() { count.set(0); totalCount.set(0); } - + @Override protected void processedEventFromQueue() { // addedEventToQueue runs in dispatcher thread while // processedEventFromQueue runs in state change notifier event handling thread. // It is not guaranteed that addedEventToQueue is invoked before processedEventFromQueue. // so sleep here until there's available events - while(count.get() <=0) { + while (count.get() <= 0) { try { Thread.sleep(10); LOG.info("sleep to wait for available events"); @@ -84,7 +84,7 @@ protected void processedEventFromQueue() { } } } - + @Override protected void addedEventToQueue() { totalCount.incrementAndGet(); @@ -117,15 +117,15 @@ public void testEventsOnRegistration() { VertexStateUpdateListener mockListener12 = mock(VertexStateUpdateListener.class); VertexStateUpdateListener mockListener13 = mock(VertexStateUpdateListener.class); VertexStateUpdateListener mockListener14 = mock(VertexStateUpdateListener.class); - // Register for all states + // Register for all states tracker.registerForVertexUpdates(v1.getName(), null, mockListener11); - // Register for all states + // Register for all states tracker.registerForVertexUpdates(v1.getName(), EnumSet.allOf( VertexState.class), mockListener12); - // Register for specific state, event generated + // Register for specific state, event generated tracker.registerForVertexUpdates(v1.getName(), EnumSet.of( VertexState.RUNNING), mockListener13); - // Register for specific state, event not generated + // Register for specific state, event not generated tracker.registerForVertexUpdates(v1.getName(), EnumSet.of( VertexState.SUCCEEDED), mockListener14); ArgumentCaptor argumentCaptor = diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/app/TestTezTaskCommunicatorManager.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/app/TestTezTaskCommunicatorManager.java index 483f9c40bb..64da6f78b3 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/app/TestTezTaskCommunicatorManager.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/app/TestTezTaskCommunicatorManager.java @@ -36,16 +36,13 @@ public class TestTezTaskCommunicatorManager { - @Test (timeout = 5000) + @Test(timeout = 5000) public void testContainerAliveOnGetTask() throws IOException { TaskCommunicatorContext context = mock(TaskCommunicatorContext.class); Configuration conf = new Configuration(false); UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf); - - - ApplicationId appId = ApplicationId.newInstance(1000, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); ContainerId containerId = createContainerId(appId, 1); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/CallableEventDispatcher.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/CallableEventDispatcher.java index a81bd688e9..a8e7ee46fe 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/CallableEventDispatcher.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/CallableEventDispatcher.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,15 +23,13 @@ public class CallableEventDispatcher implements EventHandler { - @Override - public void handle(CallableEvent event) { - try { - event.call(); - event.getCallback().onSuccess(null); - } catch (Exception e) { - event.getCallback().onFailure(e); - } + @Override + public void handle(CallableEvent event) { + try { + event.call(); + event.getCallback().onSuccess(null); + } catch (Exception e) { + event.getCallback().onFailure(e); } - } - +} diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestCommit.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestCommit.java index 436c522ef7..3a21b8f4c1 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestCommit.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestCommit.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -121,15 +121,15 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; /** - * + * * The test case of commit here are different from that in TestDAGImpl & * TestVertexImpl in that the commits here are running in separated thread. So * should need to pay some special attention. - * + * * 2 kinds of commit *

  • test XXX_OnDAGSuccess means TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS is true *
  • test XXX_OnVertexSuccess means TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS is false - * + * */ public class TestCommit { @@ -262,7 +262,7 @@ public CountingOutputCommitterConfig() { } public CountingOutputCommitterConfig(boolean throwError, - boolean blockCommit) { + boolean blockCommit) { this.throwError = throwError; this.blockCommit = blockCommit; } @@ -323,7 +323,7 @@ public void setupDAG(DAGPlan dagPlan) { taskCommunicatorManagerInterface, fsTokens, clock, "user", thh, appContext); doReturn(dag).when(appContext).getCurrentDAG(); doReturn(dispatcher.getEventHandler()).when(appContext).getEventHandler(); - ClusterInfo clusterInfo = new ClusterInfo(Resource.newInstance(8192,10)); + ClusterInfo clusterInfo = new ClusterInfo(Resource.newInstance(8192, 10)); doReturn(clusterInfo).when(appContext).getClusterInfo(); dispatcher.register(CallableEventType.class, new CallableEventDispatcher()); taskEventDispatcher = new TaskEventDispatcher(); @@ -400,7 +400,7 @@ private void waitForCommitCompleted(DAGImpl vertex, OutputKey outputKey) { // v2->v3 // vertex_group (v1, v2) private DAGPlan createDAGPlan(boolean vertexGroupCommitSucceeded, - boolean v3CommitSucceeded) throws Exception { + boolean v3CommitSucceeded) throws Exception { LOG.info("Setting up group dag plan"); int dummyTaskCount = 1; Resource dummyTaskResource = Resource.newInstance(1, 1); @@ -451,13 +451,13 @@ private DAGPlan createDAGPlan(boolean vertexGroupCommitSucceeded, // v2->v3 // vertex_group (v1, v2) has 2 shared outputs private DAGPlan createDAGPlanWith2VertexGroupOutputs(boolean vertexGroupCommitSucceeded1, - boolean vertexGroupCommitSucceeded2, boolean v3CommitSucceeded) throws Exception { + boolean vertexGroupCommitSucceeded2, boolean v3CommitSucceeded) throws Exception { LOG.info("Setting up group dag plan"); int dummyTaskCount = 1; Resource dummyTaskResource = Resource.newInstance(1, 1); org.apache.tez.dag.api.Vertex v1 = org.apache.tez.dag.api.Vertex.create( - "vertex1", ProcessorDescriptor.create("Processor"), dummyTaskCount, - dummyTaskResource); + "vertex1", ProcessorDescriptor.create("Processor"), dummyTaskCount, + dummyTaskResource); org.apache.tez.dag.api.Vertex v2 = org.apache.tez.dag.api.Vertex.create( "vertex2", ProcessorDescriptor.create("Processor"), dummyTaskCount, dummyTaskResource); @@ -473,9 +473,9 @@ private DAGPlan createDAGPlanWith2VertexGroupOutputs(boolean vertexGroupCommitSu .wrap(new CountingOutputCommitter.CountingOutputCommitterConfig( !vertexGroupCommitSucceeded1, true).toUserPayload()))); OutputCommitterDescriptor ocd2 = OutputCommitterDescriptor.create( - CountingOutputCommitter.class.getName()).setUserPayload( - UserPayload.create(ByteBuffer - .wrap(new CountingOutputCommitter.CountingOutputCommitterConfig( + CountingOutputCommitter.class.getName()).setUserPayload( + UserPayload.create(ByteBuffer + .wrap(new CountingOutputCommitter.CountingOutputCommitterConfig( !vertexGroupCommitSucceeded2, true).toUserPayload()))); OutputCommitterDescriptor ocd3 = OutputCommitterDescriptor.create( CountingOutputCommitter.class.getName()).setUserPayload( @@ -511,7 +511,7 @@ private DAGPlan createDAGPlan_SingleVertexWith2Committer( // used for route event error in VM private DAGPlan createDAGPlan_SingleVertexWith2Committer - (boolean commit1Succeed, boolean commit2Succeed, boolean customVM) throws IOException { + (boolean commit1Succeed, boolean commit2Succeed, boolean customVM) throws IOException { LOG.info("Setting up group dag plan"); int dummyTaskCount = 1; Resource dummyTaskResource = Resource.newInstance(1, 1); @@ -793,7 +793,7 @@ public void testVertexRouteEventErrorWhileCommitting() throws Exception { VertexManagerEvent vmEvent = VertexManagerEvent.create("vertex1", ByteBuffer.wrap(new byte[0])); TezTaskAttemptID taId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(v1.getVertexId(), 0), 0); TezEvent tezEvent = new TezEvent(vmEvent, - new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", + new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", null, taId)); v1.handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezEvent))); waitUntil(dag, DAGState.FAILED); @@ -1342,7 +1342,7 @@ public void testDAGCommitFail2_OnVertexSuccess() throws Exception { } // vertex group (v1,v2) succeeded first and then commit of vertex v3 fail - @Test (timeout = 5000) + @Test(timeout = 5000) public void testDAGCommitFail3_OnVertexSuccess() throws Exception { conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false); @@ -1464,7 +1464,7 @@ public void testDAGCommitFail4_OnVertexSuccess() throws Exception { Assert.assertEquals(1, v3OutputCommitter.abortCounter); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testDAGInternalErrorWhileCommiting_OnDAGSuccess() throws Exception { conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, true); @@ -1514,7 +1514,6 @@ public void testDAGInternalErrorWhileCommiting_OnDAGSuccess() throws Exception { // Assert.assertEquals(0, v3OutputCommitter.abortCounter); } - @Test(timeout = 5000) public void testDAGKilledWhileCommitting1_OnDAGSuccess() throws Exception { _testDAGTerminatedWhileCommitting1_OnDAGSuccess(DAGTerminationCause.DAG_KILL); @@ -1579,7 +1578,6 @@ private void _testDAGTerminatedWhileCommitting1_OnDAGSuccess(DAGTerminationCause Assert.assertEquals(1, v3OutputCommitter.abortCounter); } - @Test(timeout = 5000) public void testDAGKilledWhileCommitting1_OnVertexSuccess() throws Exception { _testDAGTerminatedWhileCommitting1_OnVertexSuccess(DAGTerminationCause.DAG_KILL); @@ -1827,13 +1825,13 @@ public void testDAGCommitInternalErrorWhileCommiting_OnDAGSuccess() throws Excep Assert.assertEquals(1, v3OutputCommitter.abortCounter); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testVertexGroupCommitFinishedEventFail_OnVertexSuccess() throws Exception { conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false); setupDAG(createDAGPlan(true, true)); historyEventHandler.failVertexGroupCommitFinishedEvent = true; - + initDAG(dag); startDAG(dag); VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1"); @@ -1870,7 +1868,7 @@ public void testVertexGroupCommitFinishedEventFail_OnVertexSuccess() throws Exce Assert.assertEquals(VertexState.KILLED, v3.getState()); Assert.assertEquals(VertexTerminationCause.OTHER_VERTEX_FAILURE, v3.getTerminationCause()); Assert.assertTrue(v3.commitFutures.isEmpty()); - + Assert.assertEquals(1, v12OutputCommitter.initCounter); Assert.assertEquals(1, v12OutputCommitter.setupCounter); Assert.assertEquals(1, v12OutputCommitter.commitCounter); @@ -1888,7 +1886,7 @@ public void testDAGCommitStartedEventFail_OnDAGSuccess() throws Exception { true); setupDAG(createDAGPlan(true, true)); historyEventHandler.failDAGCommitStartedEvent = true; - + initDAG(dag); startDAG(dag); VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1"); @@ -1971,7 +1969,7 @@ private void _testCommitCanceled_OnDAGSuccess(DAGTerminationCause terminationCau dag.handle(new DAGEventTerminateDag(dag.getID(), terminationCause, null)); waitUntil(dag, terminationCause.getFinishedState()); - + Assert.assertEquals(terminationCause, dag.getTerminationCause()); // mean the commits have been canceled Assert.assertTrue(dag.commitFutures.isEmpty()); @@ -1995,13 +1993,12 @@ private void _testCommitCanceled_OnDAGSuccess(DAGTerminationCause terminationCau // commit is not started because ControlledThreadPoolExecutor wait before schedule tasks Assert.assertEquals(0, v12OutputCommitter.commitCounter); Assert.assertEquals(1, v12OutputCommitter.abortCounter); - + Assert.assertEquals(1, v3OutputCommitter.initCounter); Assert.assertEquals(1, v3OutputCommitter.setupCounter); // commit is not started because ControlledThreadPoolExecutor wait before schedule tasks Assert.assertEquals(0, v3OutputCommitter.commitCounter); Assert.assertEquals(1, v3OutputCommitter.abortCounter); - } public static class FailOnVMEventReceivedlVertexManager extends ImmediateStartVertexManager { @@ -2015,7 +2012,6 @@ public void onVertexManagerEventReceived(VertexManagerEvent vmEvent) { super.onVertexManagerEventReceived(vmEvent); throw new RuntimeException("fail vm"); } - } private static class MockHistoryEventHandler extends HistoryEventHandler { @@ -2023,6 +2019,7 @@ private static class MockHistoryEventHandler extends HistoryEventHandler { public boolean failVertexGroupCommitFinishedEvent = false; public boolean failDAGCommitStartedEvent = false; public Queue historyEvents = new ConcurrentLinkedQueue(); + public MockHistoryEventHandler(AppContext context) { super(context); } @@ -2044,9 +2041,9 @@ public void verifyVertexGroupCommitStartedEvent(String groupName, int expectedTi int actualTimes = 0; for (HistoryEvent event : historyEvents) { if (event.getEventType() == HistoryEventType.VERTEX_GROUP_COMMIT_STARTED) { - VertexGroupCommitStartedEvent startedEvent = (VertexGroupCommitStartedEvent)event; + VertexGroupCommitStartedEvent startedEvent = (VertexGroupCommitStartedEvent) event; if (startedEvent.getVertexGroupName().equals(groupName)) { - actualTimes ++; + actualTimes++; } } } @@ -2057,9 +2054,9 @@ public void verifyVertexGroupCommitFinishedEvent(String groupName, int expectedT int actualTimes = 0; for (HistoryEvent event : historyEvents) { if (event.getEventType() == HistoryEventType.VERTEX_GROUP_COMMIT_FINISHED) { - VertexGroupCommitFinishedEvent finishedEvent = (VertexGroupCommitFinishedEvent)event; + VertexGroupCommitFinishedEvent finishedEvent = (VertexGroupCommitFinishedEvent) event; if (finishedEvent.getVertexGroupName().equals(groupName)) { - actualTimes ++; + actualTimes++; } } } @@ -2070,9 +2067,9 @@ public void verifyVertexCommitStartedEvent(TezVertexID vertexId, int expectedTim int actualTimes = 0; for (HistoryEvent event : historyEvents) { if (event.getEventType() == HistoryEventType.VERTEX_COMMIT_STARTED) { - VertexCommitStartedEvent startedEvent = (VertexCommitStartedEvent)event; + VertexCommitStartedEvent startedEvent = (VertexCommitStartedEvent) event; if (startedEvent.getVertexID().equals(vertexId)) { - actualTimes ++; + actualTimes++; } } } @@ -2083,9 +2080,9 @@ public void verifyVertexFinishedEvent(TezVertexID vertexId, int expectedTimes) { int actualTimes = 0; for (HistoryEvent event : historyEvents) { if (event.getEventType() == HistoryEventType.VERTEX_FINISHED) { - VertexFinishedEvent finishedEvent = (VertexFinishedEvent)event; + VertexFinishedEvent finishedEvent = (VertexFinishedEvent) event; if (finishedEvent.getVertexID().equals(vertexId)) { - actualTimes ++; + actualTimes++; } } } @@ -2096,9 +2093,9 @@ public void verifyDAGCommitStartedEvent(TezDAGID dagId, int expectedTimes) { int actualTimes = 0; for (HistoryEvent event : historyEvents) { if (event.getEventType() == HistoryEventType.DAG_COMMIT_STARTED) { - DAGCommitStartedEvent startedEvent = (DAGCommitStartedEvent)event; + DAGCommitStartedEvent startedEvent = (DAGCommitStartedEvent) event; if (startedEvent.getDagID().equals(dagId)) { - actualTimes ++; + actualTimes++; } } } @@ -2109,16 +2106,16 @@ public void verifyDAGFinishedEvent(TezDAGID dagId, int expectedTimes) { int actualTimes = 0; for (HistoryEvent event : historyEvents) { if (event.getEventType() == HistoryEventType.DAG_FINISHED) { - DAGFinishedEvent startedEvent = (DAGFinishedEvent)event; + DAGFinishedEvent startedEvent = (DAGFinishedEvent) event; if (startedEvent.getDAGID().equals(dagId)) { - actualTimes ++; + actualTimes++; } } } Assert.assertEquals(expectedTimes, actualTimes); } } - + private static class ControlledThreadPoolExecutor extends ThreadPoolExecutor { public ControlledThreadPoolExecutor(int poolSize) { @@ -2127,15 +2124,15 @@ public ControlledThreadPoolExecutor(int poolSize) { } public ControlledThreadPoolExecutor(int corePoolSize, int maximumPoolSize, - long keepAliveTime, TimeUnit unit, BlockingQueue workQueue) { + long keepAliveTime, TimeUnit unit, BlockingQueue workQueue) { super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue); } public boolean startFlag = false; - + @Override protected void beforeExecute(Thread t, Runnable r) { - while(!startFlag) { + while (!startFlag) { try { Thread.sleep(100); } catch (InterruptedException e) { diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java index 9a3292e8b8..4dd0cfe701 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -193,7 +193,7 @@ public class TestDAGImpl { private AppContext dagWithCustomEdgeAppContext; private HistoryEventHandler historyEventHandler; private TaskAttemptEventDispatcher taskAttemptEventDispatcher; - private ClusterInfo clusterInfo = new ClusterInfo(Resource.newInstance(8192,10)); + private ClusterInfo clusterInfo = new ClusterInfo(Resource.newInstance(8192, 10)); private HadoopShim defaultShim = new DefaultHadoopShim(); static { @@ -218,7 +218,7 @@ private DAGImpl chooseDAG(TezDAGID curDAGId) { + ", dagId=" + curDAGId); } } - + private class DagEventDispatcher implements EventHandler { @Override public void handle(DAGEvent event) { @@ -235,7 +235,7 @@ public void handle(TaskEvent event) { DAGImpl handler = chooseDAG(id); Vertex vertex = handler.getVertex(event.getVertexID()); Task task = vertex.getTask(event.getTaskID()); - ((EventHandler)task).handle(event); + ((EventHandler) task).handle(event); } } @@ -255,10 +255,10 @@ public void handle(TaskAttemptEvent event) { Vertex vertex = handler.getVertex(event.getVertexID()); Task task = vertex.getTask(event.getTaskID()); TaskAttempt ta = task.getAttempt(event.getTaskAttemptID()); - ((EventHandler)ta).handle(event); + ((EventHandler) ta).handle(event); } } - + private class VertexEventDispatcher implements EventHandler { @@ -273,7 +273,7 @@ public void handle(VertexEvent event) { } private class DAGFinishEventHandler - implements EventHandler { + implements EventHandler { public int dagFinishEvents = 0; @Override @@ -288,136 +288,138 @@ private DAGPlan createTestMRRDAGPlan() { .setName("testverteximpl") .addVertex( VertexPlan.newBuilder() - .setName("vertex1") - .setType(PlanVertexType.NORMAL) - .addTaskLocationHint( - PlanTaskLocationHint.newBuilder() - .addHost("host1") - .addRack("rack1") - .build() + .setName("vertex1") + .setType(PlanVertexType.NORMAL) + .addTaskLocationHint( + PlanTaskLocationHint.newBuilder() + .addHost("host1") + .addRack("rack1") + .build() ) .setTaskConfig( PlanTaskConfiguration.newBuilder() - .setNumTasks(1) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("x1.y1") - .build() - ) + .setNumTasks(1) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("x1.y1") + .build() + ) .addOutputs( DAGProtos.RootInputLeafOutputProto.newBuilder() - .setIODescriptor( - TezEntityDescriptorProto.newBuilder().setClassName("output1").build() - ) - .setName("output1") + .setIODescriptor( + TezEntityDescriptorProto.newBuilder().setClassName("output1").build() + ) + .setName("output1") .setControllerDescriptor( TezEntityDescriptorProto.newBuilder().setClassName( CountingOutputCommitter.class.getName())) - ) - .addOutEdgeId("e1") - .build() - ) + ) + .addOutEdgeId("e1") + .build() + ) .addVertex( VertexPlan.newBuilder() - .setName("vertex2") - .setType(PlanVertexType.NORMAL) - .addTaskLocationHint( - PlanTaskLocationHint.newBuilder() - .addHost("host2") - .addRack("rack2") - .build() + .setName("vertex2") + .setType(PlanVertexType.NORMAL) + .addTaskLocationHint( + PlanTaskLocationHint.newBuilder() + .addHost("host2") + .addRack("rack2") + .build() ) .setTaskConfig( PlanTaskConfiguration.newBuilder() - .setNumTasks(1) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("x2.y2") - .build() - ) + .setNumTasks(1) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("x2.y2") + .build() + ) .addOutputs( DAGProtos.RootInputLeafOutputProto.newBuilder() - .setIODescriptor( - TezEntityDescriptorProto.newBuilder().setClassName("output2").build() - ) - .setName("output2") - .setControllerDescriptor( - TezEntityDescriptorProto.newBuilder().setClassName( - CountingOutputCommitter.class.getName())) - ) - .addInEdgeId("e1") - .addOutEdgeId("e2") - .build() - ) + .setIODescriptor( + TezEntityDescriptorProto.newBuilder().setClassName("output2").build() + ) + .setName("output2") + .setControllerDescriptor( + TezEntityDescriptorProto.newBuilder().setClassName( + CountingOutputCommitter.class.getName())) + ) + .addInEdgeId("e1") + .addOutEdgeId("e2") + .build() + ) .addVertex( VertexPlan.newBuilder() - .setName("vertex3") - .setType(PlanVertexType.NORMAL) - .setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("x3.y3")) - .addTaskLocationHint( - PlanTaskLocationHint.newBuilder() - .addHost("host3") - .addRack("rack3") - .build() + .setName("vertex3") + .setType(PlanVertexType.NORMAL) + .setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("x3.y3")) + .addTaskLocationHint( + PlanTaskLocationHint.newBuilder() + .addHost("host3") + .addRack("rack3") + .build() ) .setTaskConfig( PlanTaskConfiguration.newBuilder() - .setNumTasks(1) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("foo") - .setTaskModule("x3.y3") - .build() - ) - .addOutputs( + .setNumTasks(1) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("foo") + .setTaskModule("x3.y3") + .build() + ) + .addOutputs( DAGProtos.RootInputLeafOutputProto.newBuilder() - .setIODescriptor( - TezEntityDescriptorProto.newBuilder().setClassName("output3").build() - ) - .setName("output3") - .setControllerDescriptor( - TezEntityDescriptorProto.newBuilder().setClassName( - CountingOutputCommitter.class.getName())) - ) - .addInEdgeId("e2") - .build() - ) + .setIODescriptor( + TezEntityDescriptorProto.newBuilder().setClassName("output3").build() + ) + .setName("output3") + .setControllerDescriptor( + TezEntityDescriptorProto.newBuilder().setClassName( + CountingOutputCommitter.class.getName())) + ) + .addInEdgeId("e2") + .build() + ) + .addEdge( + EdgePlan.newBuilder() + .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i2")) + .setInputVertexName("vertex1") + .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o1")) + .setOutputVertexName("vertex2") + .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) + .setId("e1") + .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) + .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) + .build() + ) .addEdge( EdgePlan.newBuilder() - .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i2")) - .setInputVertexName("vertex1") - .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o1")) - .setOutputVertexName("vertex2") - .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) - .setId("e1") - .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) - .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) - .build() - ) - .addEdge( - EdgePlan.newBuilder() - .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i3")) - .setInputVertexName("vertex2") - .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o2")) - .setOutputVertexName("vertex3") - .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) - .setId("e2") - .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) - .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) - .build() - ) - .build(); + .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i3")) + .setInputVertexName("vertex2") + .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o2")) + .setOutputVertexName("vertex3") + .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) + .setId("e2") + .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) + .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) + .build() + ) + .build(); return dag; } public static class TotalCountingOutputCommitter extends CountingOutputCommitter { static int totalCommitCounter = 0; + public TotalCountingOutputCommitter(OutputCommitterContext context) { super(context); } + @Override public void commitOutput() throws IOException { ++totalCommitCounter; @@ -473,219 +475,219 @@ public static DAGPlan createTestDAGPlan() { .setValue(3 + ""))) .addVertex( VertexPlan.newBuilder() - .setName("vertex1") - .setType(PlanVertexType.NORMAL) - .addTaskLocationHint( - PlanTaskLocationHint.newBuilder() - .addHost("host1") - .addRack("rack1") - .build() + .setName("vertex1") + .setType(PlanVertexType.NORMAL) + .addTaskLocationHint( + PlanTaskLocationHint.newBuilder() + .addHost("host1") + .addRack("rack1") + .build() ) - .setTaskConfig( - PlanTaskConfiguration.newBuilder() - .setNumTasks(1) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("x1.y1") - .build() + .setTaskConfig( + PlanTaskConfiguration.newBuilder() + .setNumTasks(1) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("x1.y1") + .build() ) - .setVertexConf(ConfigurationProto.newBuilder() - .addConfKeyValues(PlanKeyValuePair.newBuilder() - .setKey(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS) - .setValue(2+""))) - .addOutEdgeId("e1") - .build() - ) + .setVertexConf(ConfigurationProto.newBuilder() + .addConfKeyValues(PlanKeyValuePair.newBuilder() + .setKey(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS) + .setValue(2 + ""))) + .addOutEdgeId("e1") + .build() + ) .addVertex( VertexPlan.newBuilder() - .setName("vertex2") - .setType(PlanVertexType.NORMAL) - .addTaskLocationHint( - PlanTaskLocationHint.newBuilder() - .addHost("host2") - .addRack("rack2") - .build() + .setName("vertex2") + .setType(PlanVertexType.NORMAL) + .addTaskLocationHint( + PlanTaskLocationHint.newBuilder() + .addHost("host2") + .addRack("rack2") + .build() ) - .setTaskConfig( - PlanTaskConfiguration.newBuilder() - .setNumTasks(2) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("x2.y2") - .build() + .setTaskConfig( + PlanTaskConfiguration.newBuilder() + .setNumTasks(2) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("x2.y2") + .build() ) - .addOutEdgeId("e2") - .build() - ) + .addOutEdgeId("e2") + .build() + ) .addVertex( VertexPlan.newBuilder() - .setName("vertex3") - .setType(PlanVertexType.NORMAL) - .setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("x3.y3")) - .addTaskLocationHint( - PlanTaskLocationHint.newBuilder() - .addHost("host3") - .addRack("rack3") - .build() + .setName("vertex3") + .setType(PlanVertexType.NORMAL) + .setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("x3.y3")) + .addTaskLocationHint( + PlanTaskLocationHint.newBuilder() + .addHost("host3") + .addRack("rack3") + .build() ) - .setTaskConfig( - PlanTaskConfiguration.newBuilder() - .setNumTasks(2) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("foo") - .setTaskModule("x3.y3") - .build() + .setTaskConfig( + PlanTaskConfiguration.newBuilder() + .setNumTasks(2) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("foo") + .setTaskModule("x3.y3") + .build() ) - .addInEdgeId("e1") - .addInEdgeId("e2") - .addOutEdgeId("e3") - .addOutEdgeId("e4") - .build() - ) + .addInEdgeId("e1") + .addInEdgeId("e2") + .addOutEdgeId("e3") + .addOutEdgeId("e4") + .build() + ) .addVertex( VertexPlan.newBuilder() - .setName("vertex4") - .setType(PlanVertexType.NORMAL) - .addTaskLocationHint( - PlanTaskLocationHint.newBuilder() - .addHost("host4") - .addRack("rack4") - .build() + .setName("vertex4") + .setType(PlanVertexType.NORMAL) + .addTaskLocationHint( + PlanTaskLocationHint.newBuilder() + .addHost("host4") + .addRack("rack4") + .build() ) - .setTaskConfig( - PlanTaskConfiguration.newBuilder() - .setNumTasks(2) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("x4.y4") - .build() + .setTaskConfig( + PlanTaskConfiguration.newBuilder() + .setNumTasks(2) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("x4.y4") + .build() ) - .addInEdgeId("e3") - .addOutEdgeId("e5") - .build() - ) + .addInEdgeId("e3") + .addOutEdgeId("e5") + .build() + ) .addVertex( VertexPlan.newBuilder() - .setName("vertex5") - .setType(PlanVertexType.NORMAL) - .addTaskLocationHint( - PlanTaskLocationHint.newBuilder() - .addHost("host5") - .addRack("rack5") - .build() + .setName("vertex5") + .setType(PlanVertexType.NORMAL) + .addTaskLocationHint( + PlanTaskLocationHint.newBuilder() + .addHost("host5") + .addRack("rack5") + .build() ) - .setTaskConfig( - PlanTaskConfiguration.newBuilder() - .setNumTasks(2) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("x5.y5") - .build() + .setTaskConfig( + PlanTaskConfiguration.newBuilder() + .setNumTasks(2) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("x5.y5") + .build() ) - .addInEdgeId("e4") - .addOutEdgeId("e6") - .build() - ) + .addInEdgeId("e4") + .addOutEdgeId("e6") + .build() + ) .addVertex( VertexPlan.newBuilder() - .setName("vertex6") - .setType(PlanVertexType.NORMAL) - .addTaskLocationHint( - PlanTaskLocationHint.newBuilder() - .addHost("host6") - .addRack("rack6") - .build() + .setName("vertex6") + .setType(PlanVertexType.NORMAL) + .addTaskLocationHint( + PlanTaskLocationHint.newBuilder() + .addHost("host6") + .addRack("rack6") + .build() ) - .setTaskConfig( - PlanTaskConfiguration.newBuilder() - .setNumTasks(2) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("x6.y6") - .build() + .setTaskConfig( + PlanTaskConfiguration.newBuilder() + .setNumTasks(2) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("x6.y6") + .build() ) - .addInEdgeId("e5") - .addInEdgeId("e6") - .build() - ) + .addInEdgeId("e5") + .addInEdgeId("e6") + .build() + ) .addEdge( EdgePlan.newBuilder() - .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i3_v1")) - .setInputVertexName("vertex1") - .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o1")) - .setOutputVertexName("vertex3") - .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) - .setId("e1") - .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) - .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) - .build() - ) + .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i3_v1")) + .setInputVertexName("vertex1") + .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o1")) + .setOutputVertexName("vertex3") + .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) + .setId("e1") + .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) + .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) + .build() + ) .addEdge( EdgePlan.newBuilder() - .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i3_v2")) - .setInputVertexName("vertex2") - .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o2")) - .setOutputVertexName("vertex3") - .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) - .setId("e2") - .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) - .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) - .build() - ) + .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i3_v2")) + .setInputVertexName("vertex2") + .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o2")) + .setOutputVertexName("vertex3") + .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) + .setId("e2") + .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) + .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) + .build() + ) .addEdge( EdgePlan.newBuilder() - .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i4_v3")) - .setInputVertexName("vertex3") - .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o3_v4")) - .setOutputVertexName("vertex4") - .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) - .setId("e3") - .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) - .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) - .build() - ) + .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i4_v3")) + .setInputVertexName("vertex3") + .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o3_v4")) + .setOutputVertexName("vertex4") + .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) + .setId("e3") + .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) + .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) + .build() + ) .addEdge( EdgePlan.newBuilder() - .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i5_v3")) - .setInputVertexName("vertex3") - .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o3_v5")) - .setOutputVertexName("vertex5") - .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) - .setId("e4") - .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) - .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) - .build() - ) + .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i5_v3")) + .setInputVertexName("vertex3") + .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o3_v5")) + .setOutputVertexName("vertex5") + .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) + .setId("e4") + .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) + .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) + .build() + ) .addEdge( EdgePlan.newBuilder() - .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i6_v4")) - .setInputVertexName("vertex4") - .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o4")) - .setOutputVertexName("vertex6") - .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) - .setId("e5") - .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) - .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) - .build() - ) + .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i6_v4")) + .setInputVertexName("vertex4") + .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o4")) + .setOutputVertexName("vertex6") + .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) + .setId("e5") + .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) + .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) + .build() + ) .addEdge( EdgePlan.newBuilder() - .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i6_v5")) - .setInputVertexName("vertex5") - .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o5")) - .setOutputVertexName("vertex6") - .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) - .setId("e6") - .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) - .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) - .build() - ) + .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i6_v5")) + .setInputVertexName("vertex5") + .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o5")) + .setOutputVertexName("vertex6") + .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) + .setId("e6") + .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) + .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) + .build() + ) .build(); return dag; @@ -718,49 +720,49 @@ private DAGPlan createDAGWithCustomEdge(ExceptionLocation exLocation, boolean us .addOutEdgeId("e1") .build() ) - .addVertex( - VertexPlan.newBuilder() - .setName("vertex2") - .setType(PlanVertexType.NORMAL) - .setProcessorDescriptor( - TezEntityDescriptorProto.newBuilder().setClassName("x2.y2")) - .addTaskLocationHint( - PlanTaskLocationHint.newBuilder() - .addHost("host2") - .addRack("rack2") - .build() - ) - .setTaskConfig( - PlanTaskConfiguration.newBuilder() - .setNumTasks(2) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("foo") - .setTaskModule("x2.y2") - .build() - ) - .addInEdgeId("e1") - .build() - ) - .addEdge( - EdgePlan.newBuilder() - .setEdgeManager(TezEntityDescriptorProto.newBuilder() - .setClassName(useLegacy ? CustomizedEdgeManagerLegacy.class.getName() : - CustomizedEdgeManager.class.getName()) - .setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder() - .setUserPayload(ByteString.copyFromUtf8(exLocation.name()))) - ) - .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("v1_v2")) - .setInputVertexName("vertex1") - .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o1")) - .setOutputVertexName("vertex2") - .setDataMovementType(PlanEdgeDataMovementType.CUSTOM) - .setId("e1") - .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) - .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) - .build() - ) - .build(); + .addVertex( + VertexPlan.newBuilder() + .setName("vertex2") + .setType(PlanVertexType.NORMAL) + .setProcessorDescriptor( + TezEntityDescriptorProto.newBuilder().setClassName("x2.y2")) + .addTaskLocationHint( + PlanTaskLocationHint.newBuilder() + .addHost("host2") + .addRack("rack2") + .build() + ) + .setTaskConfig( + PlanTaskConfiguration.newBuilder() + .setNumTasks(2) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("foo") + .setTaskModule("x2.y2") + .build() + ) + .addInEdgeId("e1") + .build() + ) + .addEdge( + EdgePlan.newBuilder() + .setEdgeManager(TezEntityDescriptorProto.newBuilder() + .setClassName(useLegacy ? CustomizedEdgeManagerLegacy.class.getName() : + CustomizedEdgeManager.class.getName()) + .setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder() + .setUserPayload(ByteString.copyFromUtf8(exLocation.name()))) + ) + .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("v1_v2")) + .setInputVertexName("vertex1") + .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o1")) + .setOutputVertexName("vertex2") + .setDataMovementType(PlanEdgeDataMovementType.CUSTOM) + .setId("e1") + .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) + .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) + .build() + ) + .build(); return dag; } @@ -791,46 +793,46 @@ private DAGPlan createDAGWithNonExistEdgeManager() { .addOutEdgeId("e1") .build() ) - .addVertex( - VertexPlan.newBuilder() - .setName("vertex2") - .setType(PlanVertexType.NORMAL) - .setProcessorDescriptor( - TezEntityDescriptorProto.newBuilder().setClassName("x2.y2")) - .addTaskLocationHint( - PlanTaskLocationHint.newBuilder() - .addHost("host2") - .addRack("rack2") - .build() - ) - .setTaskConfig( - PlanTaskConfiguration.newBuilder() - .setNumTasks(2) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("foo") - .setTaskModule("x2.y2") - .build() - ) - .addInEdgeId("e1") - .build() - ) - .addEdge( - EdgePlan.newBuilder() - .setEdgeManager(TezEntityDescriptorProto.newBuilder() - .setClassName("non-exist-edge-manager") - ) - .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("v1_v2")) - .setInputVertexName("vertex1") - .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o1")) - .setOutputVertexName("vertex2") - .setDataMovementType(PlanEdgeDataMovementType.CUSTOM) - .setId("e1") - .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) - .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) - .build() - ) - .build(); + .addVertex( + VertexPlan.newBuilder() + .setName("vertex2") + .setType(PlanVertexType.NORMAL) + .setProcessorDescriptor( + TezEntityDescriptorProto.newBuilder().setClassName("x2.y2")) + .addTaskLocationHint( + PlanTaskLocationHint.newBuilder() + .addHost("host2") + .addRack("rack2") + .build() + ) + .setTaskConfig( + PlanTaskConfiguration.newBuilder() + .setNumTasks(2) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("foo") + .setTaskModule("x2.y2") + .build() + ) + .addInEdgeId("e1") + .build() + ) + .addEdge( + EdgePlan.newBuilder() + .setEdgeManager(TezEntityDescriptorProto.newBuilder() + .setClassName("non-exist-edge-manager") + ) + .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("v1_v2")) + .setInputVertexName("vertex1") + .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o1")) + .setOutputVertexName("vertex2") + .setDataMovementType(PlanEdgeDataMovementType.CUSTOM) + .setId("e1") + .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) + .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) + .build() + ) + .build(); return dag; } @@ -839,7 +841,7 @@ public static void beforeClass() { MockDNSToSwitchMapping.initializeMockRackResolver(); } - @SuppressWarnings({ "unchecked", "rawtypes" }) + @SuppressWarnings({"unchecked", "rawtypes"}) @Before public void setup() { conf = new Configuration(); @@ -856,16 +858,17 @@ public void setup() { final ListenableFuture mockFuture = mock(ListenableFuture.class); when(appContext.getHadoopShim()).thenReturn(defaultShim); when(appContext.getApplicationID()).thenReturn(appAttemptId.getApplicationId()); - + doAnswer(new Answer() { public ListenableFuture answer(InvocationOnMock invocation) { - Object[] args = invocation.getArguments(); - CallableEvent e = (CallableEvent) args[0]; - dispatcher.getEventHandler().handle(e); - return mockFuture; - }}) - .when(execService).submit((Callable) any()); - + Object[] args = invocation.getArguments(); + CallableEvent e = (CallableEvent) args[0]; + dispatcher.getEventHandler().handle(e); + return mockFuture; + } + }) + .when(execService).submit((Callable) any()); + doReturn(execService).when(appContext).getExecService(); historyEventHandler = mock(HistoryEventHandler.class); aclManager = new ACLManager("amUser"); @@ -936,7 +939,7 @@ public ListenableFuture answer(InvocationOnMock invocation) { dispatcher.init(conf); dispatcher.start(); } - + @After public void teardown() { dispatcher.await(); @@ -971,9 +974,9 @@ public void handle(AMSchedulerEvent event) { private void setupDAGWithCustomEdge(ExceptionLocation exLocation) { setupDAGWithCustomEdge(exLocation, false); } - + private void setupDAGWithCustomEdge(ExceptionLocation exLocation, boolean useLegacy) { - dagWithCustomEdgeId = TezDAGID.getInstance(appAttemptId.getApplicationId(), 4); + dagWithCustomEdgeId = TezDAGID.getInstance(appAttemptId.getApplicationId(), 4); dagPlanWithCustomEdge = createDAGWithCustomEdge(exLocation, useLegacy); dagWithCustomEdgeAppContext = mock(AppContext.class); doReturn(aclManager).when(dagWithCustomEdgeAppContext).getAMACLManager(); @@ -1008,7 +1011,7 @@ private void startDAG(DAGImpl impl) { dispatcher.await(); Assert.assertEquals(DAGState.RUNNING, impl.getState()); } - + @Test(timeout = 5000) public void testDAGInit() { initDAG(dag); @@ -1030,7 +1033,7 @@ public void testDAGInitFailed() { @Test(timeout = 5000) public void testDAGInitFailedDuetoInvalidResource() { // cluster maxContainerCapability is less than the vertex resource request - ClusterInfo clusterInfo = new ClusterInfo(Resource.newInstance(512,10)); + ClusterInfo clusterInfo = new ClusterInfo(Resource.newInstance(512, 10)); doReturn(clusterInfo).when(appContext).getClusterInfo(); dag.handle( new DAGEvent(dag.getID(), DAGEventType.DAG_INIT)); @@ -1047,7 +1050,7 @@ public void testDAGStart() { startDAG(dag); dispatcher.await(); - for (int i = 0 ; i < 6; ++i ) { + for (int i = 0; i < 6; ++i) { TezVertexID vId = TezVertexID.getInstance(dagId, i); Vertex v = dag.getVertex(vId); Assert.assertEquals(VertexState.RUNNING, v.getState()); @@ -1055,14 +1058,14 @@ public void testDAGStart() { Assert.assertEquals(0, v.getDistanceFromRoot()); } else if (i == 2) { Assert.assertEquals(1, v.getDistanceFromRoot()); - } else if ( i > 2 && i < 5) { + } else if (i > 2 && i < 5) { Assert.assertEquals(2, v.getDistanceFromRoot()); } else if (i == 5) { Assert.assertEquals(3, v.getDistanceFromRoot()); } } - for (int i = 0 ; i < 6; ++i ) { + for (int i = 0; i < 6; ++i) { TezVertexID vId = TezVertexID.getInstance(dagId, i); LOG.info("Distance from root: v" + i + ":" + dag.getVertex(vId).getDistanceFromRoot()); @@ -1073,7 +1076,7 @@ public void testDAGStart() { public void testNonExistEdgeManagerPlugin() { dagPlan = createDAGWithNonExistEdgeManager(); dag = new DAGImpl(dagId, conf, dagPlan, - dispatcher.getEventHandler(), taskCommunicatorManagerInterface, + dispatcher.getEventHandler(), taskCommunicatorManagerInterface, fsTokens, clock, "user", thh, appContext); dag.entityUpdateTracker = new StateChangeNotifierForTest(dag); doReturn(dag).when(appContext).getCurrentDAG(); @@ -1085,11 +1088,11 @@ public void testNonExistEdgeManagerPlugin() { .contains("java.lang.ClassNotFoundException: non-exist-edge-manager")); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testNonExistDAGScheduler() { conf.set(TezConfiguration.TEZ_AM_DAG_SCHEDULER_CLASS, "non-exist-dag-scheduler"); dag = new DAGImpl(dagId, conf, dagPlan, - dispatcher.getEventHandler(), taskCommunicatorManagerInterface, + dispatcher.getEventHandler(), taskCommunicatorManagerInterface, fsTokens, clock, "user", thh, appContext); dag.entityUpdateTracker = new StateChangeNotifierForTest(dag); doReturn(dag).when(appContext).getCurrentDAG(); @@ -1126,7 +1129,7 @@ public void testVertexCompletion() { Assert.assertEquals((float) 2 / 11, dag.getCompletedTaskProgress(), 0.05); } - + @SuppressWarnings("unchecked") @Test(timeout = 5000) public void testEdgeManager_GetNumDestinationTaskPhysicalInputs() { @@ -1137,7 +1140,7 @@ public void testEdgeManager_GetNumDestinationTaskPhysicalInputs() { null)); dispatcher.await(); - VertexImpl v2 = (VertexImpl)dagWithCustomEdge.getVertex("vertex2"); + VertexImpl v2 = (VertexImpl) dagWithCustomEdge.getVertex("vertex2"); String diag = StringUtils.join(v2.getDiagnostics(), ","); Assert.assertTrue(diag.contains(ExceptionLocation.GetNumDestinationTaskPhysicalInputs.name())); } @@ -1155,30 +1158,30 @@ public void testEdgeManager_GetNumSourceTaskPhysicalOutputs() { // dag failed. Assert.assertEquals(DAGState.FAILED, dagWithCustomEdge.getState()); - VertexImpl v1 = (VertexImpl)dagWithCustomEdge.getVertex("vertex1"); + VertexImpl v1 = (VertexImpl) dagWithCustomEdge.getVertex("vertex1"); String diag = StringUtils.join(v1.getDiagnostics(), ","); Assert.assertTrue(diag.contains(ExceptionLocation.GetNumSourceTaskPhysicalOutputs.name())); } - + @SuppressWarnings("unchecked") @Test(timeout = 5000) public void testEdgeManager_RouteDataMovementEventToDestination() { setupDAGWithCustomEdge(ExceptionLocation.RouteDataMovementEventToDestination); dispatcher.getEventHandler().handle( new DAGEvent(dagWithCustomEdge.getID(), DAGEventType.DAG_INIT)); - dispatcher.getEventHandler().handle(new DAGEventStartDag(dagWithCustomEdge.getID(), + dispatcher.getEventHandler().handle(new DAGEventStartDag(dagWithCustomEdge.getID(), null)); dispatcher.await(); Assert.assertEquals(DAGState.RUNNING, dagWithCustomEdge.getState()); - VertexImpl v1 = (VertexImpl)dagWithCustomEdge.getVertex("vertex1"); - VertexImpl v2 = (VertexImpl)dagWithCustomEdge.getVertex("vertex2"); + VertexImpl v1 = (VertexImpl) dagWithCustomEdge.getVertex("vertex1"); + VertexImpl v2 = (VertexImpl) dagWithCustomEdge.getVertex("vertex2"); dispatcher.await(); - Task t1= v2.getTask(0); - TaskAttemptImpl ta1= (TaskAttemptImpl)t1.getAttempt(TezTaskAttemptID.getInstance(t1.getTaskID(), 0)); + Task t1 = v2.getTask(0); + TaskAttemptImpl ta1 = (TaskAttemptImpl) t1.getAttempt(TezTaskAttemptID.getInstance(t1.getTaskID(), 0)); DataMovementEvent daEvent = DataMovementEvent.create(ByteBuffer.wrap(new byte[0])); - TezEvent tezEvent = new TezEvent(daEvent, + TezEvent tezEvent = new TezEvent(daEvent, new EventMetaData(EventProducerConsumerType.INPUT, "vertex1", "vertex2", ta1.getTaskAttemptID())); dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v2.getVertexId(), Lists.newArrayList(tezEvent))); dispatcher.await(); @@ -1190,7 +1193,7 @@ public void testEdgeManager_RouteDataMovementEventToDestination() { String diag = StringUtils.join(v2.getDiagnostics(), ","); Assert.assertTrue(diag.contains(ExceptionLocation.RouteDataMovementEventToDestination.name())); } - + @SuppressWarnings("unchecked") @Test(timeout = 5000) public void testEdgeManager_RouteDataMovementEventToDestinationWithLegacyRouting() { @@ -1203,15 +1206,15 @@ public void testEdgeManager_RouteDataMovementEventToDestinationWithLegacyRouting dispatcher.await(); Assert.assertEquals(DAGState.RUNNING, dagWithCustomEdge.getState()); - VertexImpl v1 = (VertexImpl)dagWithCustomEdge.getVertex("vertex1"); - VertexImpl v2 = (VertexImpl)dagWithCustomEdge.getVertex("vertex2"); + VertexImpl v1 = (VertexImpl) dagWithCustomEdge.getVertex("vertex1"); + VertexImpl v2 = (VertexImpl) dagWithCustomEdge.getVertex("vertex2"); dispatcher.await(); - Task t1= v2.getTask(0); - TaskAttemptImpl ta1= (TaskAttemptImpl)t1.getAttempt(TezTaskAttemptID.getInstance(t1.getTaskID(), 0)); + Task t1 = v2.getTask(0); + TaskAttemptImpl ta1 = (TaskAttemptImpl) t1.getAttempt(TezTaskAttemptID.getInstance(t1.getTaskID(), 0)); DataMovementEvent daEvent = DataMovementEvent.create(ByteBuffer.wrap(new byte[0])); - TezEvent tezEvent = new TezEvent(daEvent, + TezEvent tezEvent = new TezEvent(daEvent, new EventMetaData(EventProducerConsumerType.INPUT, "vertex1", "vertex2", ta1.getTaskAttemptID())); dispatcher.getEventHandler().handle( new VertexEventRouteEvent(v2.getVertexId(), Lists.newArrayList(tezEvent))); @@ -1230,26 +1233,26 @@ public void testEdgeManager_RouteInputSourceTaskFailedEventToDestinationLegacyRo setupDAGWithCustomEdge(ExceptionLocation.RouteInputSourceTaskFailedEventToDestination, true); dispatcher.getEventHandler().handle( new DAGEvent(dagWithCustomEdge.getID(), DAGEventType.DAG_INIT)); - dispatcher.getEventHandler().handle(new DAGEventStartDag(dagWithCustomEdge.getID(), + dispatcher.getEventHandler().handle(new DAGEventStartDag(dagWithCustomEdge.getID(), null)); dispatcher.await(); Assert.assertEquals(DAGState.RUNNING, dagWithCustomEdge.getState()); - VertexImpl v1 = (VertexImpl)dagWithCustomEdge.getVertex("vertex1"); - VertexImpl v2 = (VertexImpl)dagWithCustomEdge.getVertex("vertex2"); + VertexImpl v1 = (VertexImpl) dagWithCustomEdge.getVertex("vertex1"); + VertexImpl v2 = (VertexImpl) dagWithCustomEdge.getVertex("vertex2"); dispatcher.await(); - Task t1= v2.getTask(0); - TaskAttemptImpl ta1= (TaskAttemptImpl)t1.getAttempt(TezTaskAttemptID.getInstance(t1.getTaskID(), 0)); + Task t1 = v2.getTask(0); + TaskAttemptImpl ta1 = (TaskAttemptImpl) t1.getAttempt(TezTaskAttemptID.getInstance(t1.getTaskID(), 0)); InputFailedEvent ifEvent = InputFailedEvent.create(0, 1); - TezEvent tezEvent = new TezEvent(ifEvent, - new EventMetaData(EventProducerConsumerType.INPUT,"vertex1", "vertex2", ta1.getTaskAttemptID())); + TezEvent tezEvent = new TezEvent(ifEvent, + new EventMetaData(EventProducerConsumerType.INPUT, "vertex1", "vertex2", ta1.getTaskAttemptID())); dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v2.getVertexId(), Lists.newArrayList(tezEvent))); dispatcher.await(); v2.getTaskAttemptTezEvents(ta1.getTaskAttemptID(), 0, 0, 1000); dispatcher.await(); Assert.assertEquals(VertexState.FAILED, v2.getState()); - + Assert.assertEquals(VertexState.KILLED, v1.getState()); String diag = StringUtils.join(v2.getDiagnostics(), ","); Assert.assertTrue(diag.contains(ExceptionLocation.RouteInputSourceTaskFailedEventToDestination.name())); @@ -1266,16 +1269,16 @@ public void testEdgeManager_GetNumDestinationConsumerTasks() { dispatcher.await(); Assert.assertEquals(DAGState.RUNNING, dagWithCustomEdge.getState()); - VertexImpl v1 = (VertexImpl)dagWithCustomEdge.getVertex("vertex1"); - VertexImpl v2 = (VertexImpl)dagWithCustomEdge.getVertex("vertex2"); + VertexImpl v1 = (VertexImpl) dagWithCustomEdge.getVertex("vertex1"); + VertexImpl v2 = (VertexImpl) dagWithCustomEdge.getVertex("vertex2"); dispatcher.await(); - Task t1= v2.getTask(0); - TaskAttemptImpl ta1= (TaskAttemptImpl)t1.getAttempt(TezTaskAttemptID.getInstance(t1.getTaskID(), 0)); + Task t1 = v2.getTask(0); + TaskAttemptImpl ta1 = (TaskAttemptImpl) t1.getAttempt(TezTaskAttemptID.getInstance(t1.getTaskID(), 0)); InputReadErrorEvent ireEvent = InputReadErrorEvent.create("", 0, 0); - TezEvent tezEvent = new TezEvent(ireEvent, - new EventMetaData(EventProducerConsumerType.INPUT,"vertex2", "vertex1", ta1.getTaskAttemptID())); + TezEvent tezEvent = new TezEvent(ireEvent, + new EventMetaData(EventProducerConsumerType.INPUT, "vertex2", "vertex1", ta1.getTaskAttemptID())); dispatcher.getEventHandler().handle( new VertexEventRouteEvent(v2.getVertexId(), Lists.newArrayList(tezEvent))); dispatcher.await(); @@ -1285,27 +1288,27 @@ public void testEdgeManager_GetNumDestinationConsumerTasks() { String diag = StringUtils.join(v2.getDiagnostics(), ","); Assert.assertTrue(diag.contains(ExceptionLocation.GetNumDestinationConsumerTasks.name())); } - + @SuppressWarnings("unchecked") @Test(timeout = 5000) public void testEdgeManager_RouteInputErrorEventToSource() { setupDAGWithCustomEdge(ExceptionLocation.RouteInputErrorEventToSource); dispatcher.getEventHandler().handle( new DAGEvent(dagWithCustomEdge.getID(), DAGEventType.DAG_INIT)); - dispatcher.getEventHandler().handle(new DAGEventStartDag(dagWithCustomEdge.getID(), + dispatcher.getEventHandler().handle(new DAGEventStartDag(dagWithCustomEdge.getID(), null)); dispatcher.await(); Assert.assertEquals(DAGState.RUNNING, dagWithCustomEdge.getState()); - VertexImpl v1 = (VertexImpl)dagWithCustomEdge.getVertex("vertex1"); - VertexImpl v2 = (VertexImpl)dagWithCustomEdge.getVertex("vertex2"); + VertexImpl v1 = (VertexImpl) dagWithCustomEdge.getVertex("vertex1"); + VertexImpl v2 = (VertexImpl) dagWithCustomEdge.getVertex("vertex2"); dispatcher.await(); - Task t1= v2.getTask(0); - TaskAttemptImpl ta1= (TaskAttemptImpl)t1.getAttempt(TezTaskAttemptID.getInstance(t1.getTaskID(), 0)); + Task t1 = v2.getTask(0); + TaskAttemptImpl ta1 = (TaskAttemptImpl) t1.getAttempt(TezTaskAttemptID.getInstance(t1.getTaskID(), 0)); InputReadErrorEvent ireEvent = InputReadErrorEvent.create("", 0, 0); - TezEvent tezEvent = new TezEvent(ireEvent, - new EventMetaData(EventProducerConsumerType.INPUT,"vertex2", "vertex1", ta1.getTaskAttemptID())); + TezEvent tezEvent = new TezEvent(ireEvent, + new EventMetaData(EventProducerConsumerType.INPUT, "vertex2", "vertex1", ta1.getTaskAttemptID())); dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v2.getVertexId(), Lists.newArrayList(tezEvent))); dispatcher.await(); // @@ -1314,7 +1317,7 @@ public void testEdgeManager_RouteInputErrorEventToSource() { String diag = StringUtils.join(v2.getDiagnostics(), ","); Assert.assertTrue(diag.contains(ExceptionLocation.RouteInputErrorEventToSource.name())); } - + @SuppressWarnings("unchecked") @Test(timeout = 5000) public void testGroupDAGCompletionWithCommitSuccess() { @@ -1323,20 +1326,20 @@ public void testGroupDAGCompletionWithCommitSuccess() { startDAG(groupDag); dispatcher.await(); - for (int i=0; i<3; ++i) { - Vertex v = groupDag.getVertex("vertex"+(i+1)); + for (int i = 0; i < 3; ++i) { + Vertex v = groupDag.getVertex("vertex" + (i + 1)); dispatcher.getEventHandler().handle(new VertexEventTaskCompleted( TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED)); dispatcher.await(); Assert.assertEquals(VertexState.SUCCEEDED, v.getState()); - Assert.assertEquals(i+1, groupDag.getSuccessfulVertices()); + Assert.assertEquals(i + 1, groupDag.getSuccessfulVertices()); } - + Assert.assertEquals(3, groupDag.getSuccessfulVertices()); Assert.assertTrue(1.0f == groupDag.getCompletedTaskProgress()); Assert.assertEquals(DAGState.SUCCEEDED, groupDag.getState()); Assert.assertEquals(2, TotalCountingOutputCommitter.totalCommitCounter); - } + } @SuppressWarnings("unchecked") @Test(timeout = 5000) @@ -1395,26 +1398,26 @@ public void testDAGCompletionWithCommitSuccess() { dispatcher.await(); startDAG(mrrDag); dispatcher.await(); - for (int i=0; i<2; ++i) { - Vertex v = mrrDag.getVertex("vertex"+(i+1)); + for (int i = 0; i < 2; ++i) { + Vertex v = mrrDag.getVertex("vertex" + (i + 1)); dispatcher.getEventHandler().handle(new VertexEventTaskCompleted( TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED)); dispatcher.await(); Assert.assertEquals(VertexState.SUCCEEDED, v.getState()); - Assert.assertEquals(i+1, mrrDag.getSuccessfulVertices()); + Assert.assertEquals(i + 1, mrrDag.getSuccessfulVertices()); } - + // no commit yet for (Vertex v : mrrDag.vertices.values()) { for (OutputCommitter c : v.getOutputCommitters().values()) { - CountingOutputCommitter committer= (CountingOutputCommitter) c; + CountingOutputCommitter committer = (CountingOutputCommitter) c; Assert.assertEquals(0, committer.abortCounter); Assert.assertEquals(0, committer.commitCounter); Assert.assertEquals(1, committer.initCounter); Assert.assertEquals(1, committer.setupCounter); } } - + // dag completion and commit Vertex v = mrrDag.getVertex("vertex3"); dispatcher.getEventHandler().handle(new VertexEventTaskCompleted( @@ -1423,10 +1426,10 @@ public void testDAGCompletionWithCommitSuccess() { Assert.assertEquals(VertexState.SUCCEEDED, v.getState()); Assert.assertEquals(3, mrrDag.getSuccessfulVertices()); Assert.assertEquals(DAGState.SUCCEEDED, mrrDag.getState()); - + for (Vertex vertex : mrrDag.vertices.values()) { for (OutputCommitter c : vertex.getOutputCommitters().values()) { - CountingOutputCommitter committer= (CountingOutputCommitter) c; + CountingOutputCommitter committer = (CountingOutputCommitter) c; Assert.assertEquals(0, committer.abortCounter); Assert.assertEquals(1, committer.commitCounter); Assert.assertEquals(1, committer.initCounter); @@ -1434,9 +1437,9 @@ public void testDAGCompletionWithCommitSuccess() { } } } - + @SuppressWarnings("unchecked") - @Test(timeout=5000) + @Test(timeout = 5000) public void testDAGCompletionWithCommitFailure() throws IOException { // all vertices completed -> DAG completion and commit initDAG(mrrDag); @@ -1461,29 +1464,29 @@ public void testDAGCompletionWithCommitFailure() throws IOException { ) .build()); badVertex.setAdditionalOutputs(outputs); - + startDAG(mrrDag); dispatcher.await(); - for (int i=0; i<2; ++i) { - Vertex v = mrrDag.getVertex("vertex"+(i+1)); + for (int i = 0; i < 2; ++i) { + Vertex v = mrrDag.getVertex("vertex" + (i + 1)); dispatcher.getEventHandler().handle(new VertexEventTaskCompleted( TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED)); dispatcher.await(); Assert.assertEquals(VertexState.SUCCEEDED, v.getState()); - Assert.assertEquals(i+1, mrrDag.getSuccessfulVertices()); + Assert.assertEquals(i + 1, mrrDag.getSuccessfulVertices()); } - + // no commit yet for (Vertex v : mrrDag.vertices.values()) { for (OutputCommitter c : v.getOutputCommitters().values()) { - CountingOutputCommitter committer= (CountingOutputCommitter) c; + CountingOutputCommitter committer = (CountingOutputCommitter) c; Assert.assertEquals(0, committer.abortCounter); Assert.assertEquals(0, committer.commitCounter); Assert.assertEquals(1, committer.initCounter); Assert.assertEquals(1, committer.setupCounter); } } - + // dag completion and commit. Exception causes all outputs to be aborted Vertex v = mrrDag.getVertex("vertex3"); dispatcher.getEventHandler().handle(new VertexEventTaskCompleted( @@ -1493,45 +1496,45 @@ public void testDAGCompletionWithCommitFailure() throws IOException { Assert.assertEquals(3, mrrDag.getSuccessfulVertices()); Assert.assertEquals(DAGState.FAILED, mrrDag.getState()); Assert.assertEquals(DAGTerminationCause.COMMIT_FAILURE, mrrDag.getTerminationCause()); - + for (Vertex vertex : mrrDag.vertices.values()) { for (OutputCommitter c : vertex.getOutputCommitters().values()) { - CountingOutputCommitter committer= (CountingOutputCommitter) c; + CountingOutputCommitter committer = (CountingOutputCommitter) c; Assert.assertEquals(1, committer.abortCounter); Assert.assertEquals(1, committer.initCounter); Assert.assertEquals(1, committer.setupCounter); } } } - + @SuppressWarnings("unchecked") - @Test(timeout=5000) + @Test(timeout = 5000) public void testDAGErrorAbortAllOutputs() { // error on a vertex -> dag error -> all outputs aborted. initDAG(mrrDag); dispatcher.await(); startDAG(mrrDag); dispatcher.await(); - for (int i=0; i<2; ++i) { - Vertex v = mrrDag.getVertex("vertex"+(i+1)); + for (int i = 0; i < 2; ++i) { + Vertex v = mrrDag.getVertex("vertex" + (i + 1)); dispatcher.getEventHandler().handle(new VertexEventTaskCompleted( TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED)); dispatcher.await(); Assert.assertEquals(VertexState.SUCCEEDED, v.getState()); - Assert.assertEquals(i+1, mrrDag.getSuccessfulVertices()); + Assert.assertEquals(i + 1, mrrDag.getSuccessfulVertices()); } - + // no commit yet for (Vertex v : mrrDag.vertices.values()) { for (OutputCommitter c : v.getOutputCommitters().values()) { - CountingOutputCommitter committer= (CountingOutputCommitter) c; + CountingOutputCommitter committer = (CountingOutputCommitter) c; Assert.assertEquals(0, committer.abortCounter); Assert.assertEquals(0, committer.commitCounter); Assert.assertEquals(1, committer.initCounter); Assert.assertEquals(1, committer.setupCounter); } } - + // vertex error -> dag error -> abort all outputs Vertex v = mrrDag.getVertex("vertex3"); dispatcher.getEventHandler().handle(new VertexEvent( @@ -1539,10 +1542,10 @@ public void testDAGErrorAbortAllOutputs() { dispatcher.await(); Assert.assertEquals(VertexState.ERROR, v.getState()); Assert.assertEquals(DAGState.ERROR, mrrDag.getState()); - + for (Vertex vertex : mrrDag.vertices.values()) { for (OutputCommitter c : vertex.getOutputCommitters().values()) { - CountingOutputCommitter committer= (CountingOutputCommitter) c; + CountingOutputCommitter committer = (CountingOutputCommitter) c; Assert.assertEquals(1, committer.abortCounter); Assert.assertEquals(0, committer.commitCounter); Assert.assertEquals(1, committer.initCounter); @@ -1552,7 +1555,7 @@ public void testDAGErrorAbortAllOutputs() { } @SuppressWarnings("unchecked") - @Test(timeout=5000) + @Test(timeout = 5000) public void testDAGErrorAbortNonSuccessfulOutputs() { // vertex success -> vertex output commit. failed dag aborts only non-successful vertices mrrDag.getConf().setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false); @@ -1560,35 +1563,35 @@ public void testDAGErrorAbortNonSuccessfulOutputs() { dispatcher.await(); startDAG(mrrDag); dispatcher.await(); - for (int i=0; i<2; ++i) { - Vertex v = mrrDag.getVertex("vertex"+(i+1)); + for (int i = 0; i < 2; ++i) { + Vertex v = mrrDag.getVertex("vertex" + (i + 1)); dispatcher.getEventHandler().handle(new VertexEventTaskCompleted( TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED)); dispatcher.await(); Assert.assertEquals(VertexState.SUCCEEDED, v.getState()); - Assert.assertEquals(i+1, mrrDag.getSuccessfulVertices()); + Assert.assertEquals(i + 1, mrrDag.getSuccessfulVertices()); for (OutputCommitter c : v.getOutputCommitters().values()) { - CountingOutputCommitter committer= (CountingOutputCommitter) c; + CountingOutputCommitter committer = (CountingOutputCommitter) c; Assert.assertEquals(0, committer.abortCounter); Assert.assertEquals(1, committer.commitCounter); Assert.assertEquals(1, committer.initCounter); Assert.assertEquals(1, committer.setupCounter); } } - + // error on vertex -> dag error Vertex errorVertex = mrrDag.getVertex("vertex3"); dispatcher.getEventHandler().handle(new VertexEvent( errorVertex.getVertexId(), VertexEventType.V_INTERNAL_ERROR)); dispatcher.await(); Assert.assertEquals(VertexState.ERROR, errorVertex.getState()); - + dispatcher.await(); Assert.assertEquals(DAGState.ERROR, mrrDag.getState()); - + for (Vertex vertex : mrrDag.vertices.values()) { for (OutputCommitter c : vertex.getOutputCommitters().values()) { - CountingOutputCommitter committer= (CountingOutputCommitter) c; + CountingOutputCommitter committer = (CountingOutputCommitter) c; if (vertex == errorVertex) { Assert.assertEquals(1, committer.abortCounter); Assert.assertEquals(0, committer.commitCounter); @@ -1599,14 +1602,14 @@ public void testDAGErrorAbortNonSuccessfulOutputs() { Assert.assertEquals(1, committer.abortCounter); Assert.assertEquals(1, committer.commitCounter); Assert.assertEquals(1, committer.initCounter); - Assert.assertEquals(1, committer.setupCounter); + Assert.assertEquals(1, committer.setupCounter); } } } } @SuppressWarnings("unchecked") - @Test(timeout=5000) + @Test(timeout = 5000) public void testVertexReRunning() { initDAG(dag); dag.dagScheduler = mock(DAGScheduler.class); @@ -1624,22 +1627,21 @@ public void testVertexReRunning() { Assert.assertEquals(VertexState.SUCCEEDED, v.getState()); Assert.assertEquals(1, dag.getSuccessfulVertices()); Assert.assertEquals(1, dag.numCompletedVertices); - + dispatcher.getEventHandler().handle( new VertexEventTaskReschedule(TezTaskID.getInstance(vId, 0))); dispatcher.await(); Assert.assertEquals(VertexState.RUNNING, v.getState()); Assert.assertEquals(0, dag.getSuccessfulVertices()); Assert.assertEquals(0, dag.numCompletedVertices); - + dispatcher.getEventHandler().handle(new VertexEventTaskCompleted( TezTaskID.getInstance(vId, 0), TaskState.SUCCEEDED)); - dispatcher.await(); + dispatcher.await(); Assert.assertEquals(VertexState.SUCCEEDED, v.getState()); Assert.assertEquals(1, dag.getSuccessfulVertices()); Assert.assertEquals(1, dag.numCompletedVertices); - } @SuppressWarnings("unchecked") @@ -1652,12 +1654,11 @@ public void testKillStartedDAG() { dispatcher.await(); Assert.assertEquals(DAGState.KILLED, dag.getState()); - for (int i = 0 ; i < 6; ++i ) { + for (int i = 0; i < 6; ++i) { TezVertexID vId = TezVertexID.getInstance(dagId, i); Vertex v = dag.getVertex(vId); Assert.assertEquals(VertexState.KILLED, v.getState()); } - } @Test(timeout = 5000) @@ -1695,7 +1696,7 @@ private void _testTerminateRunningDAG(DAGTerminationCause terminationCause) { Assert.assertEquals(DAGState.TERMINATING, dag.getState()); Assert.assertEquals(VertexState.SUCCEEDED, v0.getState()); Assert.assertEquals(VertexState.TERMINATING, v1.getState()); - for (int i = 2 ; i < 6; ++i ) { + for (int i = 2; i < 6; ++i) { TezVertexID vId = TezVertexID.getInstance(dagId, i); Vertex v = dag.getVertex(vId); Assert.assertEquals(VertexState.KILLED, v.getState()); @@ -1791,7 +1792,6 @@ public void testGetDAGStatusReturnOnDagError() throws InterruptedException, TezE runTestGetDAGStatusReturnOnDagFinished(DAGStatus.State.ERROR); } - @SuppressWarnings("unchecked") public void runTestGetDAGStatusReturnOnDagFinished(DAGStatusBuilder.State testState) throws TezException, InterruptedException { initDAG(dag); @@ -1856,12 +1856,11 @@ public void runTestGetDAGStatusReturnOnDagFinished(DAGStatusBuilder.State testSt long diff = statusCheckRunnable.dagStatusEndTime - statusCheckRunnable.dagStatusStartTime; Assert.assertNotNull(statusCheckRunnable.dagStatus); Assert.assertTrue("Status: " + statusCheckRunnable.dagStatus.getState() - + ", Diff:" + diff, diff >= 0 && diff < 3500); + + ", Diff:" + diff, diff >= 0 && diff < 3500); Assert.assertEquals(testState, statusCheckRunnable.dagStatus.getState()); t1.join(); } - @SuppressWarnings("unchecked") @Test(timeout = 5000) public void testVertexFailureHandling() { @@ -1890,7 +1889,6 @@ public void testVertexFailureHandling() { } } - @Test(timeout = 5000) public void testDAGKill() { _testDAGTerminate(DAGTerminationCause.DAG_KILL); @@ -1937,7 +1935,7 @@ private void _testDAGTerminate(DAGTerminationCause terminationCause) { Assert.assertEquals(1, dagFinishEventHandler.dagFinishEvents); } - @Test (timeout = 5000L) + @Test(timeout = 5000L) @SuppressWarnings("unchecked") public void testDAGHang() throws Exception { conf.setBoolean( @@ -1974,7 +1972,7 @@ public void testDAGHang() throws Exception { dispatcher.await(); Assert.assertEquals(DAGState.COMMITTING, dag.getState()); DAGEventCommitCompleted dagEvent = new DAGEventCommitCompleted( - dagId, outputKey, false , new RuntimeException("test")); + dagId, outputKey, false, new RuntimeException("test")); doThrow(new RuntimeException("test")).when( dag).logJobHistoryUnsuccesfulEvent(any(), any()); dag.handle(dagEvent); @@ -2109,7 +2107,7 @@ public static class CustomizedEdgeManagerLegacy extends EdgeManagerPlugin { public static EdgeManagerPluginDescriptor getUserPayload(ExceptionLocation exLocation) { return EdgeManagerPluginDescriptor.create(CustomizedEdgeManager.class.getName()) - .setUserPayload(UserPayload.create(ByteBuffer.wrap(exLocation.name().getBytes()))); + .setUserPayload(UserPayload.create(ByteBuffer.wrap(exLocation.name().getBytes()))); } public CustomizedEdgeManagerLegacy(EdgeManagerPluginContext context) { @@ -2145,8 +2143,8 @@ public int getNumSourceTaskPhysicalOutputs(int sourceTaskIndex) @Override public void routeDataMovementEventToDestination(DataMovementEvent event, - int sourceTaskIndex, int sourceOutputIndex, - Map> destinationTaskAndInputIndices) + int sourceTaskIndex, int sourceOutputIndex, + Map> destinationTaskAndInputIndices) throws Exception { if (exLocation == ExceptionLocation.RouteDataMovementEventToDestination) { throw new Exception(exLocation.name()); @@ -2174,7 +2172,7 @@ public int getNumDestinationConsumerTasks(int sourceTaskIndex) @Override public int routeInputErrorEventToSource(InputReadErrorEvent event, - int destinationTaskIndex, int destinationFailedInputIndex) + int destinationTaskIndex, int destinationFailedInputIndex) throws Exception { if (exLocation == ExceptionLocation.RouteInputErrorEventToSource) { throw new Exception(exLocation.name()); @@ -2188,7 +2186,7 @@ public static class CustomizedEdgeManager extends EdgeManagerPluginOnDemand { public static EdgeManagerPluginDescriptor getUserPayload(ExceptionLocation exLocation) { return EdgeManagerPluginDescriptor.create(CustomizedEdgeManager.class.getName()) - .setUserPayload(UserPayload.create(ByteBuffer.wrap(exLocation.name().getBytes()))); + .setUserPayload(UserPayload.create(ByteBuffer.wrap(exLocation.name().getBytes()))); } public CustomizedEdgeManager(EdgeManagerPluginContext context) { @@ -2233,7 +2231,7 @@ public int getNumDestinationConsumerTasks(int sourceTaskIndex) @Override public int routeInputErrorEventToSource(int destinationTaskIndex, - int destinationFailedInputIndex) throws Exception { + int destinationFailedInputIndex) throws Exception { if (exLocation == ExceptionLocation.RouteInputErrorEventToSource) { throw new Exception(exLocation.name()); } @@ -2242,7 +2240,7 @@ public int routeInputErrorEventToSource(int destinationTaskIndex, @Override public EventRouteMetadata routeDataMovementEventToDestination(int sourceTaskIndex, - int sourceOutputIndex, int destinationTaskIndex) throws Exception { + int sourceOutputIndex, int destinationTaskIndex) throws Exception { if (exLocation == ExceptionLocation.RouteDataMovementEventToDestination) { throw new Exception(exLocation.name()); } @@ -2273,7 +2271,6 @@ public void prepareForRouting() throws Exception { } } - // Specificially for testGetDAGStatusReturnOnDagSuccess private class DagStatusCheckRunnable implements Runnable { @@ -2328,25 +2325,23 @@ public void testCounterLimits() { dispatcher.await(); startDAG(mrrDag); dispatcher.await(); - for (int i=0; i<3; ++i) { - Vertex v = mrrDag.getVertex("vertex"+(i+1)); + for (int i = 0; i < 3; ++i) { + Vertex v = mrrDag.getVertex("vertex" + (i + 1)); TezCounters ctrs = new TezCounters(); for (int j = 0; j < 50; ++j) { ctrs.findCounter("g", "c" + i + "_" + j).increment(1); } ((VertexImpl) v).setCounters(ctrs); dispatcher.getEventHandler().handle(new VertexEventTaskCompleted( - TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED)); + TezTaskID.getInstance(v.getVertexId(), 0), TaskState.SUCCEEDED)); dispatcher.await(); Assert.assertEquals(VertexState.SUCCEEDED, v.getState()); - Assert.assertEquals(i+1, mrrDag.getSuccessfulVertices()); + Assert.assertEquals(i + 1, mrrDag.getSuccessfulVertices()); } Assert.assertEquals(3, mrrDag.getSuccessfulVertices()); Assert.assertEquals(DAGState.FAILED, mrrDag.getState()); Assert.assertTrue("Diagnostics should contain counter limits error message", StringUtils.join(mrrDag.getDiagnostics(), ",").contains("Counters limit exceeded")); - } - } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java index 159dd9fb7f..6353071f36 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -196,7 +196,7 @@ public class TestDAGRecovery { private long t1FinishedTime = t1StartedTime + rand.nextInt(100); private long ta1LaunchTime = t1StartedTime + rand.nextInt(100); private long ta1FinishedTime = ta1LaunchTime + rand.nextInt(100); - + private class DagEventDispatcher implements EventHandler { @Override public void handle(DAGEvent event) { @@ -251,7 +251,7 @@ private class AMSchedulerEventDispatcher implements EventHandler historyEvents = new ArrayList(); @@ -273,7 +273,7 @@ public void verifyHistoryEvent(int expectedTimes, HistoryEventType eventType) { int actualCount = 0; for (HistoryEvent event : historyEvents) { if (event.getEventType() == eventType) { - actualCount ++; + actualCount++; } } assertEquals(expectedTimes, actualCount); @@ -289,7 +289,7 @@ public MockInputInitializer(InputInitializerContext initializerContext) { @Override public List initialize() throws Exception { // sleep forever, block the initialization - while(true) { + while (true) { Thread.sleep(1000); } } @@ -305,7 +305,7 @@ public static void beforeClass() { MockDNSToSwitchMapping.initializeMockRackResolver(); } - @SuppressWarnings({ "unchecked", "rawtypes" }) + @SuppressWarnings({"unchecked", "rawtypes"}) @Before public void setup() { conf = new Configuration(); @@ -351,7 +351,7 @@ public ListenableFuture answer(InvocationOnMock invocation) { dag.entityUpdateTracker = new StateChangeNotifierForTest(dag); doReturn(dag).when(appContext).getCurrentDAG(); ugi = mock(UserGroupInformation.class); - UserGroupInformation ugi =dag.getDagUGI(); + UserGroupInformation ugi = dag.getDagUGI(); doReturn(clusterInfo).when(appContext).getClusterInfo(); TaskSchedulerManager mockTaskScheduler = mock(TaskSchedulerManager.class); doReturn(mockTaskScheduler).when(appContext).getTaskScheduler(); @@ -383,7 +383,6 @@ public ListenableFuture answer(InvocationOnMock invocation) { LogManager.getRootLogger().setLevel(Level.DEBUG); } - public static class RecoveryNotSupportedOutputCommitter extends OutputCommitter { public RecoveryNotSupportedOutputCommitter( @@ -406,7 +405,7 @@ public void commitOutput() throws Exception { @Override public void abortOutput(State finalState) throws Exception { } - + @Override public boolean isTaskRecoverySupported() { return false; @@ -528,7 +527,6 @@ private DAGPlan createDAGPlan() { return dag; } - @After public void teardown() { dispatcher.await(); @@ -540,14 +538,14 @@ public void teardown() { } dag = null; } - - + ////////////////////////////////// DAG Recovery /////////////////////////////////////////////////// + /** * RecoveryEvents: SummaryEvent_DAGFinishedEvent(SUCCEEDED) * Recover dag to SUCCEEDED and all of its vertices to SUCCEEDED */ - @Test(timeout=5000) + @Test(timeout = 5000) public void testDAGRecoverFromDesiredSucceeded() { DAGEventRecoverEvent recoveryEvent = new DAGEventRecoverEvent(dagId, DAGState.SUCCEEDED, dagRecoveryData); dag.handle(recoveryEvent); @@ -560,12 +558,12 @@ public void testDAGRecoverFromDesiredSucceeded() { assertEquals(VertexState.SUCCEEDED, dag.getVertex("vertex3").getState()); // DAG#initTime, startTime is not guaranteed to be recovered in this case } - + /** * RecoveryEvents: SummaryEvent_DAGFinishedEvent(FAILED) * Recover dag to FAILED and all of its vertices to FAILED */ - @Test(timeout=5000) + @Test(timeout = 5000) public void testDAGRecoverFromDesiredFailed() { DAGEventRecoverEvent recoveryEvent = new DAGEventRecoverEvent(dagId, DAGState.FAILED, dagRecoveryData); dag.handle(recoveryEvent); @@ -578,12 +576,12 @@ public void testDAGRecoverFromDesiredFailed() { assertEquals(VertexState.FAILED, dag.getVertex("vertex3").getState()); // DAG#initTime, startTime is not guaranteed to be recovered in this case } - + /** * RecoveryEvents: SummaryEvent_DAGFinishedEvent(KILLED) * Recover dag to KILLED and all of its vertices to KILLED */ - @Test(timeout=5000) + @Test(timeout = 5000) public void testDAGRecoverFromDesiredKilled() { DAGEventRecoverEvent recoveryEvent = new DAGEventRecoverEvent(dagId, DAGState.KILLED, dagRecoveryData); dag.handle(recoveryEvent); @@ -596,12 +594,12 @@ public void testDAGRecoverFromDesiredKilled() { assertEquals(VertexState.KILLED, dag.getVertex("vertex3").getState()); // DAG#initTime, startTime is not guaranteed to be recovered in this case } - + /** * RecoveryEvents: SummaryEvent_DAGFinishedEvent(ERROR) * Recover dag to ERROR and all of its vertices to ERROR */ - @Test(timeout=5000) + @Test(timeout = 5000) public void testDAGRecoverFromDesiredError() { DAGEventRecoverEvent recoveryEvent = new DAGEventRecoverEvent(dagId, DAGState.ERROR, dagRecoveryData); dag.handle(recoveryEvent); @@ -614,12 +612,12 @@ public void testDAGRecoverFromDesiredError() { assertEquals(VertexState.ERROR, dag.getVertex("vertex3").getState()); // DAG#initTime, startTime is not guaranteed to be recovered in this case } - + /** * RecoveryEvents: DAGSubmittedEvent * Recover it as normal dag execution */ - @Test(timeout=5000) + @Test(timeout = 5000) public void testDAGRecoverFromNew() { DAGEventRecoverEvent recoveryEvent = new DAGEventRecoverEvent(dagId, dagRecoveryData); dag.handle(recoveryEvent); @@ -627,14 +625,14 @@ public void testDAGRecoverFromNew() { assertEquals(DAGState.RUNNING, dag.getState()); } - + /** * RecoveryEvents: DAGSubmittedEvent, DAGInitializedEvent * Recover it as normal dag execution */ - @Test(timeout=5000) + @Test(timeout = 5000) public void testDAGRecoverFromInited() { - DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagId, dagInitedTime, + DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagId, dagInitedTime, "user", "dagName", null); doReturn(dagInitedEvent).when(dagRecoveryData).getDAGInitializedEvent(); DAGEventRecoverEvent recoveryEvent = new DAGEventRecoverEvent(dagId, dagRecoveryData); @@ -644,10 +642,10 @@ public void testDAGRecoverFromInited() { assertEquals(DAGState.RUNNING, dag.getState()); assertEquals(dagInitedTime, dag.initTime); } - - @Test(timeout=5000) + + @Test(timeout = 5000) public void testDAGRecoverFromStarted() { - DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagId, dagInitedTime, + DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagId, dagInitedTime, "user", "dagName", null); doReturn(dagInitedEvent).when(dagRecoveryData).getDAGInitializedEvent(); DAGStartedEvent dagStartedEvent = new DAGStartedEvent(dagId, dagStartedTime, "user", "dagName"); @@ -661,11 +659,11 @@ public void testDAGRecoverFromStarted() { assertEquals(dagInitedTime, dag.initTime); assertEquals(dagStartedTime, dag.startTime); } - + /////////////////////////////// Vertex Recovery ///////////////////////////////////////// - - private void initMockDAGRecoveryDataForVertex() { - DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagId, dagInitedTime, + + private void initMockDAGRecoveryDataForVertex() { + DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagId, dagInitedTime, "user", "dagName", null); DAGStartedEvent dagStartedEvent = new DAGStartedEvent(dagId, dagStartedTime, "user", "dagName"); doReturn(dagInitedEvent).when(dagRecoveryData).getDAGInitializedEvent(); @@ -676,79 +674,79 @@ private void initMockDAGRecoveryDataForVertex() { * RecoveryEvents: * DAG: DAGInitedEvent -> DAGStartedEvent * V1: No any event - * + * * Reinitialize V1 again. */ - @Test(timeout=5000) + @Test(timeout = 5000) public void testVertexRecoverFromNew() { initMockDAGRecoveryDataForVertex(); - + DAGEventRecoverEvent recoveryEvent = new DAGEventRecoverEvent(dagId, dagRecoveryData); dag.handle(recoveryEvent); dispatcher.await(); assertEquals(DAGState.RUNNING, dag.getState()); // reinitialize v1 again - VertexImpl v1 = (VertexImpl)dag.getVertex("vertex1"); - VertexImpl v2 = (VertexImpl)dag.getVertex("vertex2"); - VertexImpl v3 = (VertexImpl)dag.getVertex("vertex3"); + VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1"); + VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2"); + VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3"); assertEquals(VertexState.INITIALIZING, v1.getState()); assertEquals(VertexState.RUNNING, v2.getState()); assertEquals(VertexState.INITED, v3.getState()); } - + /** * RecoveryEvents: * DAG: DAGInitedEvent -> DAGStartedEvent * V1: VertexInitializedEvent - * + * * Reinitialize V1 again. */ - @Test(timeout=5000) + @Test(timeout = 5000) public void testVertexRecoverFromInited() { initMockDAGRecoveryDataForVertex(); List inputGeneratedTezEvents = new ArrayList(); - VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, - "vertex1", 0L, v1InitedTime, + VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, + "vertex1", 0L, v1InitedTime, v1NumTask, "", null, inputGeneratedTezEvents, null); VertexRecoveryData vertexRecoveryData = new VertexRecoveryData(v1InitedEvent, null, null, null, null, false); doReturn(vertexRecoveryData).when(dagRecoveryData).getVertexRecoveryData(v1Id); - + DAGEventRecoverEvent recoveryEvent = new DAGEventRecoverEvent(dagId, dagRecoveryData); dag.handle(recoveryEvent); dispatcher.await(); assertEquals(DAGState.RUNNING, dag.getState()); // reinitialize v1 again because its VertexManager is not completed - VertexImpl v1 = (VertexImpl)dag.getVertex("vertex1"); - VertexImpl v2 = (VertexImpl)dag.getVertex("vertex2"); - VertexImpl v3 = (VertexImpl)dag.getVertex("vertex3"); + VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1"); + VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2"); + VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3"); assertEquals(VertexState.INITIALIZING, v1.getState()); assertEquals(VertexState.RUNNING, v2.getState()); assertEquals(VertexState.INITED, v3.getState()); } - + /** * RecoveryEvents: * DAG: DAGInitedEvent -> DAGStartedEvent * V1: VertexReconfigrationDoneEvent -> VertexInitializedEvent - * + * * Reinitialize V1 again. */ @Test//(timeout=5000) public void testVertexRecoverFromInitedAndReconfigureDone() { initMockDAGRecoveryDataForVertex(); List inputGeneratedTezEvents = new ArrayList(); - VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, - "vertex1", 0L, v1InitedTime, + VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, + "vertex1", 0L, v1InitedTime, v1NumTask, "", null, inputGeneratedTezEvents, null); - VertexConfigurationDoneEvent v1ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v1Id, + VertexConfigurationDoneEvent v1ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v1Id, 0L, v1NumTask, null, null, null, false); VertexRecoveryData vertexRecoveryData = new VertexRecoveryData(v1InitedEvent, v1ReconfigureDoneEvent, null, null, new HashMap(), false); doReturn(vertexRecoveryData).when(dagRecoveryData).getVertexRecoveryData(v1Id); - + DAGEventRecoverEvent recoveryEvent = new DAGEventRecoverEvent(dagId, dagRecoveryData); dag.handle(recoveryEvent); dispatcher.await(); @@ -757,9 +755,9 @@ public void testVertexRecoverFromInitedAndReconfigureDone() { } catch (InterruptedException e) { e.printStackTrace(); } - VertexImpl v1 = (VertexImpl)dag.getVertex("vertex1"); - VertexImpl v2 = (VertexImpl)dag.getVertex("vertex2"); - VertexImpl v3 = (VertexImpl)dag.getVertex("vertex3"); + VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1"); + VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2"); + VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3"); assertEquals(DAGState.RUNNING, dag.getState()); // reinitialize v1 assertEquals(VertexState.INITIALIZING, v1.getState()); @@ -774,7 +772,7 @@ public void testVertexRecoverFromInitedAndReconfigureDone() { * * Reinitialize V1 again. */ - @Test(timeout=5000) + @Test(timeout = 5000) public void testVertexRecoverFromStart() { initMockDAGRecoveryDataForVertex(); List inputGeneratedTezEvents = new ArrayList(); @@ -792,9 +790,9 @@ public void testVertexRecoverFromStart() { dag.handle(recoveryEvent); dispatcher.await(); - VertexImpl v1 = (VertexImpl)dag.getVertex("vertex1"); - VertexImpl v2 = (VertexImpl)dag.getVertex("vertex2"); - VertexImpl v3 = (VertexImpl)dag.getVertex("vertex3"); + VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1"); + VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2"); + VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3"); assertEquals(DAGState.RUNNING, dag.getState()); // reinitialize v1 assertEquals(VertexState.INITIALIZING, v1.getState()); @@ -809,7 +807,7 @@ public void testVertexRecoverFromStart() { * * V1 skip initialization. */ - @Test(timeout=5000) + @Test(timeout = 5000) public void testVertexRecoverWithSetParallelismCalledFlag() { initMockDAGRecoveryDataForVertex(); List inputGeneratedTezEvents = new ArrayList(); @@ -827,9 +825,9 @@ public void testVertexRecoverWithSetParallelismCalledFlag() { dag.handle(recoveryEvent); dispatcher.await(); - VertexImpl v1 = (VertexImpl)dag.getVertex("vertex1"); - VertexImpl v2 = (VertexImpl)dag.getVertex("vertex2"); - VertexImpl v3 = (VertexImpl)dag.getVertex("vertex3"); + VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1"); + VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2"); + VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3"); assertEquals(DAGState.RUNNING, dag.getState()); // v1 skip initialization assertEquals(VertexState.RUNNING, v1.getState()); @@ -847,7 +845,7 @@ public void testVertexRecoverWithSetParallelismCalledFlag() { * * V1 skip initialization. */ - @Test(timeout=5000) + @Test(timeout = 5000) public void testVertexRecoverFromVertexTaskStart() { initMockDAGRecoveryDataForVertex(); List inputGeneratedTezEvents = new ArrayList(); @@ -872,9 +870,9 @@ public void testVertexRecoverFromVertexTaskStart() { dag.handle(recoveryEvent); dispatcher.await(); - VertexImpl v1 = (VertexImpl)dag.getVertex("vertex1"); - VertexImpl v2 = (VertexImpl)dag.getVertex("vertex2"); - VertexImpl v3 = (VertexImpl)dag.getVertex("vertex3"); + VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1"); + VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2"); + VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3"); assertEquals(DAGState.RUNNING, dag.getState()); // v1 skip initialization assertEquals(VertexState.RUNNING, v1.getState()); @@ -896,7 +894,7 @@ public void testVertexRecoverFromVertexTaskStart() { * V2 skip initialization. * V3 skip initialization. */ - @Test(timeout=5000) + @Test(timeout = 5000) public void testMultipleVertexRecoverFromVertexTaskStart() { initMockDAGRecoveryDataForVertex(); List inputGeneratedTezEvents = new ArrayList(); @@ -954,9 +952,9 @@ public void testMultipleVertexRecoverFromVertexTaskStart() { dag.handle(recoveryEvent); dispatcher.await(); - VertexImpl v1 = (VertexImpl)dag.getVertex("vertex1"); - VertexImpl v2 = (VertexImpl)dag.getVertex("vertex2"); - VertexImpl v3 = (VertexImpl)dag.getVertex("vertex3"); + VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1"); + VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2"); + VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3"); assertEquals(DAGState.RUNNING, dag.getState()); @@ -990,7 +988,7 @@ public void testMultipleVertexRecoverFromVertexTaskStart() { * V2 skip initialization. * Reinitialize V3 again. Since V3 is dependent on V1 */ - @Test(timeout=5000) + @Test(timeout = 5000) public void testMultipleVertexRecoverFromVertex() { initMockDAGRecoveryDataForVertex(); List inputGeneratedTezEvents = new ArrayList(); @@ -1039,9 +1037,9 @@ public void testMultipleVertexRecoverFromVertex() { dag.handle(recoveryEvent); dispatcher.await(); - VertexImpl v1 = (VertexImpl)dag.getVertex("vertex1"); - VertexImpl v2 = (VertexImpl)dag.getVertex("vertex2"); - VertexImpl v3 = (VertexImpl)dag.getVertex("vertex3"); + VertexImpl v1 = (VertexImpl) dag.getVertex("vertex1"); + VertexImpl v2 = (VertexImpl) dag.getVertex("vertex2"); + VertexImpl v3 = (VertexImpl) dag.getVertex("vertex3"); assertEquals(DAGState.RUNNING, dag.getState()); // reinitialize v1 @@ -1056,16 +1054,16 @@ public void testMultipleVertexRecoverFromVertex() { // reinitialize v3 assertEquals(VertexState.INITED, v3.getState()); } - + /////////////////////////////// Task //////////////////////////////////////////////////////////// - + private void initMockDAGRecoveryDataForTask() { List inputGeneratedTezEvents = new ArrayList(); - VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, - "vertex1", 0L, v1InitedTime, + VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, + "vertex1", 0L, v1InitedTime, v1NumTask, "", null, inputGeneratedTezEvents, null); Map rootInputSpecs = new HashMap(); - VertexConfigurationDoneEvent v1ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v1Id, + VertexConfigurationDoneEvent v1ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v1Id, 0L, v1NumTask, null, null, rootInputSpecs, true); VertexStartedEvent v1StartedEvent = new VertexStartedEvent(v1Id, 0L, v1StartedTime); TaskStartedEvent v1taskStartedEvent = new TaskStartedEvent(t1v1Id, "vertex1", 0L, 0L); @@ -1075,8 +1073,8 @@ private void initMockDAGRecoveryDataForTask() { v1taskRecoveryDataMap.put(t1v1Id, v1taskRecoveryData); VertexRecoveryData v1RecoveryData = new VertexRecoveryData(v1InitedEvent, v1ReconfigureDoneEvent, v1StartedEvent, null, v1taskRecoveryDataMap, false); - - DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagId, dagInitedTime, + + DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagId, dagInitedTime, "user", "dagName", null); DAGStartedEvent dagStartedEvent = new DAGStartedEvent(dagId, dagStartedTime, "user", "dagName"); doReturn(v1RecoveryData).when(dagRecoveryData).getVertexRecoveryData(v1Id); @@ -1088,61 +1086,61 @@ private void initMockDAGRecoveryDataForTask() { * RecoveryEvent: TaskFinishedEvent(KILLED) * Recover it to KILLED */ - @Test(timeout=5000) + @Test(timeout = 5000) public void testTaskRecoverFromKilled() { initMockDAGRecoveryDataForTask(); TaskFinishedEvent taskFinishedEvent = new TaskFinishedEvent(t1v1Id, "v1", 0L, 0L, null, TaskState.KILLED, "", null, 4); TaskRecoveryData taskRecoveryData = new TaskRecoveryData(null, taskFinishedEvent, null); doReturn(taskRecoveryData).when(dagRecoveryData).getTaskRecoveryData(t1v1Id); - + dag.handle(new DAGEventRecoverEvent(dagId, dagRecoveryData)); dispatcher.await(); VertexImpl vertex1 = (VertexImpl) dag.getVertex(v1Id); - TaskImpl task = (TaskImpl)vertex1.getTask(t1v1Id); + TaskImpl task = (TaskImpl) vertex1.getTask(t1v1Id); assertEquals(TaskStateInternal.KILLED, task.getInternalState()); assertEquals(1, vertex1.getCompletedTasks()); } - + /** * RecoveryEvent: TaskStartedEvent * Recover it to Scheduled */ - @Test(timeout=5000) + @Test(timeout = 5000) public void testTaskRecoverFromStarted() { initMockDAGRecoveryDataForTask(); TaskStartedEvent taskStartedEvent = new TaskStartedEvent(t1v1Id, "v1", 0L, 0L); TaskRecoveryData taskRecoveryData = new TaskRecoveryData(taskStartedEvent, null, null); doReturn(taskRecoveryData).when(dagRecoveryData).getTaskRecoveryData(t1v1Id); - + dag.handle(new DAGEventRecoverEvent(dagId, dagRecoveryData)); dispatcher.await(); - + VertexImpl vertex1 = (VertexImpl) dag.getVertex(v1Id); - TaskImpl task = (TaskImpl)vertex1.getTask(t1v1Id); + TaskImpl task = (TaskImpl) vertex1.getTask(t1v1Id); assertEquals(TaskStateInternal.SCHEDULED, task.getInternalState()); } - + /** * RecoveryEvent: TaskStartedEvent -> TaskFinishedEvent * Recover it to Scheduled */ - @Test(timeout=5000) + @Test(timeout = 5000) public void testTaskRecoverFromSucceeded() { initMockDAGRecoveryDataForTask(); TaskStartedEvent taskStartedEvent = new TaskStartedEvent(t1v1Id, "v1", 0L, 0L); TaskFinishedEvent taskFinishedEvent = new TaskFinishedEvent(t1v1Id, "v1", 0L, 0L, null, TaskState.SUCCEEDED, "", null, 4); TaskAttemptStartedEvent taStartedEvent = new TaskAttemptStartedEvent( - ta1t1v1Id, "v1", 0L, mock(ContainerId.class), + ta1t1v1Id, "v1", 0L, mock(ContainerId.class), mock(NodeId.class), "", "", ""); List taGeneratedEvents = new ArrayList(); EventMetaData metadata = new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", "vertex3", ta1t1v2Id); taGeneratedEvents.add(new TezEvent(DataMovementEvent.create(ByteBuffer.wrap(new byte[0])), metadata)); TaskAttemptFinishedEvent taFinishedEvent = new TaskAttemptFinishedEvent( - ta1t1v1Id, "v1", 0L, 0L, + ta1t1v1Id, "v1", 0L, 0L, TaskAttemptState.SUCCEEDED, null, null, "", null, null, taGeneratedEvents, 0L, null, 0L, null, null, null, null, null); TaskAttemptRecoveryData taRecoveryData = new TaskAttemptRecoveryData(taStartedEvent, taFinishedEvent); @@ -1155,10 +1153,10 @@ public void testTaskRecoverFromSucceeded() { dag.handle(new DAGEventRecoverEvent(dagId, dagRecoveryData)); dispatcher.await(); - + VertexImpl vertex1 = (VertexImpl) dag.getVertex(v1Id); - TaskImpl task = (TaskImpl)vertex1.getTask(t1v1Id); - TaskAttemptImpl taskAttempt = (TaskAttemptImpl)task.getAttempt(ta1t1v1Id); + TaskImpl task = (TaskImpl) vertex1.getTask(t1v1Id); + TaskAttemptImpl taskAttempt = (TaskAttemptImpl) task.getAttempt(ta1t1v1Id); assertEquals(VertexState.RUNNING, vertex1.getState()); assertEquals(1, vertex1.getCompletedTasks()); assertEquals(TaskStateInternal.SUCCEEDED, task.getInternalState()); @@ -1166,25 +1164,25 @@ public void testTaskRecoverFromSucceeded() { } /////////////////////////////// TaskAttempt Recovery ///////////////////////////////////////////////////// - + private void initMockDAGRecoveryDataForTaskAttempt() { TaskStartedEvent t1StartedEvent = new TaskStartedEvent(t1v1Id, "vertex1", 0L, t1StartedTime); TaskRecoveryData taskRecoveryData = new TaskRecoveryData(t1StartedEvent, null, null); Map taskRecoveryDataMap = new HashMap(); taskRecoveryDataMap.put(t1v1Id, taskRecoveryData); - + List inputGeneratedTezEvents = new ArrayList(); - VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, - "vertex1", 0L, v1InitedTime, + VertexInitializedEvent v1InitedEvent = new VertexInitializedEvent(v1Id, + "vertex1", 0L, v1InitedTime, v1NumTask, "", null, inputGeneratedTezEvents, null); Map rootInputSpecs = new HashMap(); - VertexConfigurationDoneEvent v1ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v1Id, + VertexConfigurationDoneEvent v1ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v1Id, 0L, v1NumTask, null, null, rootInputSpecs, true); VertexStartedEvent v1StartedEvent = new VertexStartedEvent(v1Id, 0L, v1StartedTime); VertexRecoveryData v1RecoveryData = new VertexRecoveryData(v1InitedEvent, v1ReconfigureDoneEvent, v1StartedEvent, null, taskRecoveryDataMap, false); - - DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagId, dagInitedTime, + + DAGInitializedEvent dagInitedEvent = new DAGInitializedEvent(dagId, dagInitedTime, "user", "dagName", null); DAGStartedEvent dagStartedEvent = new DAGStartedEvent(dagId, dagStartedTime, "user", "dagName"); doReturn(v1RecoveryData).when(dagRecoveryData).getVertexRecoveryData(v1Id); @@ -1196,21 +1194,21 @@ private void initMockDAGRecoveryDataForTaskAttempt() { * RecoveryEvents: TaskAttemptFinishedEvent (FAILED) * Recover it to FAILED */ - @Test(timeout=5000) + @Test(timeout = 5000) public void testTARecoverFromNewToFailed() { initMockDAGRecoveryDataForTaskAttempt(); TaskAttemptFinishedEvent taFinishedEvent = new TaskAttemptFinishedEvent( - ta1t1v1Id, "v1", ta1LaunchTime, ta1FinishedTime, + ta1t1v1Id, "v1", ta1LaunchTime, ta1FinishedTime, TaskAttemptState.FAILED, TaskFailureType.NON_FATAL, TaskAttemptTerminationCause.CONTAINER_LAUNCH_FAILED, "", null, null, null, 0L, null, 0L, null, null, null, null, null); TaskAttemptRecoveryData taRecoveryData = new TaskAttemptRecoveryData(null, taFinishedEvent); doReturn(taRecoveryData).when(dagRecoveryData).getTaskAttemptRecoveryData(ta1t1v1Id); - + dag.handle(new DAGEventRecoverEvent(dagId, dagRecoveryData)); dispatcher.await(); - TaskImpl task = (TaskImpl)dag.getVertex(v1Id).getTask(t1v1Id); - TaskAttemptImpl taskAttempt = (TaskAttemptImpl)task.getAttempt(ta1t1v1Id); + TaskImpl task = (TaskImpl) dag.getVertex(v1Id).getTask(t1v1Id); + TaskAttemptImpl taskAttempt = (TaskAttemptImpl) task.getAttempt(ta1t1v1Id); assertEquals(TaskAttemptStateInternal.FAILED, taskAttempt.getInternalState()); assertEquals(TaskAttemptTerminationCause.CONTAINER_LAUNCH_FAILED, taskAttempt.getTerminationCause()); historyEventHandler.verifyHistoryEvent(0, HistoryEventType.TASK_ATTEMPT_STARTED); @@ -1220,26 +1218,26 @@ public void testTARecoverFromNewToFailed() { assertEquals(2, task.getAttempts().size()); assertEquals(ta1FinishedTime, taskAttempt.getFinishTime()); } - + /** * RecoveryEvents: TaskAttemptFinishedEvent (KILLED) * Recover it to KILLED */ - @Test(timeout=5000) + @Test(timeout = 5000) public void testTARecoverFromNewToKilled() { initMockDAGRecoveryDataForTaskAttempt(); TaskAttemptFinishedEvent taFinishedEvent = new TaskAttemptFinishedEvent( - ta1t1v1Id, "v1", ta1LaunchTime, ta1FinishedTime, + ta1t1v1Id, "v1", ta1LaunchTime, ta1FinishedTime, TaskAttemptState.KILLED, null, TaskAttemptTerminationCause.TERMINATED_BY_CLIENT, "", null, null, null, 0L, null, 0L, null, null, null, null, null); TaskAttemptRecoveryData taRecoveryData = new TaskAttemptRecoveryData(null, taFinishedEvent); doReturn(taRecoveryData).when(dagRecoveryData).getTaskAttemptRecoveryData(ta1t1v1Id); - + dag.handle(new DAGEventRecoverEvent(dagId, dagRecoveryData)); dispatcher.await(); - TaskImpl task = (TaskImpl)dag.getVertex(v1Id).getTask(t1v1Id); - TaskAttemptImpl taskAttempt = (TaskAttemptImpl)task.getAttempt(ta1t1v1Id); + TaskImpl task = (TaskImpl) dag.getVertex(v1Id).getTask(t1v1Id); + TaskAttemptImpl taskAttempt = (TaskAttemptImpl) task.getAttempt(ta1t1v1Id); assertEquals(TaskAttemptStateInternal.KILLED, taskAttempt.getInternalState()); assertEquals(TaskAttemptTerminationCause.TERMINATED_BY_CLIENT, taskAttempt.getTerminationCause()); historyEventHandler.verifyHistoryEvent(0, HistoryEventType.TASK_ATTEMPT_STARTED); @@ -1247,25 +1245,25 @@ public void testTARecoverFromNewToKilled() { assertEquals(0, task.failedAttempts); assertEquals(ta1FinishedTime, taskAttempt.getFinishTime()); } - + /** * RecoveryEvents: TaskAttemptStartedEvent * Recover it to KILLED */ - @Test(timeout=5000) + @Test(timeout = 5000) public void testTARecoverFromRunning() { initMockDAGRecoveryDataForTaskAttempt(); TaskAttemptStartedEvent taStartedEvent = new TaskAttemptStartedEvent( - ta1t1v1Id, "v1", ta1LaunchTime, mock(ContainerId.class), + ta1t1v1Id, "v1", ta1LaunchTime, mock(ContainerId.class), mock(NodeId.class), "", "", ""); TaskAttemptRecoveryData taRecoveryData = new TaskAttemptRecoveryData(taStartedEvent, null); doReturn(taRecoveryData).when(dagRecoveryData).getTaskAttemptRecoveryData(ta1t1v1Id); - + dag.handle(new DAGEventRecoverEvent(dagId, dagRecoveryData)); dispatcher.await(); - TaskImpl task = (TaskImpl)dag.getVertex(v1Id).getTask(t1v1Id); - TaskAttemptImpl taskAttempt = (TaskAttemptImpl)task.getAttempt(ta1t1v1Id); + TaskImpl task = (TaskImpl) dag.getVertex(v1Id).getTask(t1v1Id); + TaskAttemptImpl taskAttempt = (TaskAttemptImpl) task.getAttempt(ta1t1v1Id); assertEquals(TaskAttemptStateInternal.KILLED, taskAttempt.getInternalState()); assertEquals(TaskAttemptTerminationCause.TERMINATED_AT_RECOVERY, taskAttempt.getTerminationCause()); historyEventHandler.verifyHistoryEvent(0, HistoryEventType.TASK_ATTEMPT_STARTED); @@ -1277,11 +1275,11 @@ public void testTARecoverFromRunning() { * RecoveryEvents: TaskAttemptStartedEvent -> TaskAttemptFinishedEvent (SUCCEEDED) * Recover it to SUCCEEDED */ - @Test(timeout=5000) + @Test(timeout = 5000) public void testTARecoverFromSucceeded() { initMockDAGRecoveryDataForTaskAttempt(); TaskAttemptStartedEvent taStartedEvent = new TaskAttemptStartedEvent( - ta1t1v1Id, "v1", ta1LaunchTime, mock(ContainerId.class), + ta1t1v1Id, "v1", ta1LaunchTime, mock(ContainerId.class), mock(NodeId.class), "", "", ""); List taGeneratedEvents = new ArrayList(); EventMetaData sourceInfo = new EventMetaData(EventProducerConsumerType.INPUT, "vertex1", @@ -1289,7 +1287,7 @@ public void testTARecoverFromSucceeded() { taGeneratedEvents.add(new TezEvent(DataMovementEvent.create(ByteBuffer.wrap(new byte[0])), sourceInfo)); TaskAttemptFinishedEvent taFinishedEvent = new TaskAttemptFinishedEvent( - ta1t1v1Id, "v1", ta1LaunchTime, ta1FinishedTime, + ta1t1v1Id, "v1", ta1LaunchTime, ta1FinishedTime, TaskAttemptState.SUCCEEDED, null, null, "", null, null, taGeneratedEvents, 0L, null, 0L, null, null, null, null, null); TaskAttemptRecoveryData taRecoveryData = new TaskAttemptRecoveryData(taStartedEvent, taFinishedEvent); @@ -1297,9 +1295,9 @@ public void testTARecoverFromSucceeded() { dag.handle(new DAGEventRecoverEvent(dagId, dagRecoveryData)); dispatcher.await(); - - TaskImpl task = (TaskImpl)dag.getVertex(v1Id).getTask(t1v1Id); - TaskAttemptImpl taskAttempt = (TaskAttemptImpl)task.getAttempt(ta1t1v1Id); + + TaskImpl task = (TaskImpl) dag.getVertex(v1Id).getTask(t1v1Id); + TaskAttemptImpl taskAttempt = (TaskAttemptImpl) task.getAttempt(ta1t1v1Id); assertEquals(TaskAttemptStateInternal.SUCCEEDED, taskAttempt.getInternalState()); historyEventHandler.verifyHistoryEvent(0, HistoryEventType.TASK_ATTEMPT_FINISHED); assertEquals(TaskStateInternal.SUCCEEDED, task.getInternalState()); @@ -1313,51 +1311,50 @@ public void testTARecoverFromSucceeded() { * V2's committer is not recovery supported */ @Test//(timeout=5000) - public void testTARecoverFromSucceeded_OutputCommitterRecoveryNotSupported() throws Exception{ + public void testTARecoverFromSucceeded_OutputCommitterRecoveryNotSupported() throws Exception { initMockDAGRecoveryDataForTaskAttempt(); // set up v2 recovery data // ta1t1v2: TaskAttemptStartedEvent -> TaskAttemptFinishedEvent(SUCCEEDED) // t1v2: TaskStartedEvent // v2: VertexInitializedEvent -> VertexConfigurationDoneEvent -> VertexStartedEvent TaskAttemptStartedEvent taStartedEvent = new TaskAttemptStartedEvent( - ta1t1v2Id, "vertex2", ta1LaunchTime, mock(ContainerId.class), + ta1t1v2Id, "vertex2", ta1LaunchTime, mock(ContainerId.class), mock(NodeId.class), "", "", ""); List taGeneratedEvents = new ArrayList(); EventMetaData metadata = new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex2", "vertex3", ta1t1v2Id); taGeneratedEvents.add(new TezEvent(DataMovementEvent.create(ByteBuffer.wrap(new byte[0])), metadata)); TaskAttemptFinishedEvent taFinishedEvent = new TaskAttemptFinishedEvent( - ta1t1v2Id, "vertex2", ta1LaunchTime, ta1FinishedTime, + ta1t1v2Id, "vertex2", ta1LaunchTime, ta1FinishedTime, TaskAttemptState.SUCCEEDED, null, null, "", null, null, taGeneratedEvents, 0L, null, 0L, null, null, null, null, null); TaskAttemptRecoveryData taRecoveryData = new TaskAttemptRecoveryData(taStartedEvent, taFinishedEvent); - doReturn(taRecoveryData).when(dagRecoveryData).getTaskAttemptRecoveryData(ta1t1v2Id); + doReturn(taRecoveryData).when(dagRecoveryData).getTaskAttemptRecoveryData(ta1t1v2Id); Map taRecoveryDataMap = new HashMap(); taRecoveryDataMap.put(ta1t1v2Id, taRecoveryData); - + TaskStartedEvent t1StartedEvent = new TaskStartedEvent(t1v2Id, "vertex2", 0L, t1StartedTime); TaskRecoveryData taskRecoveryData = new TaskRecoveryData(t1StartedEvent, null, taRecoveryDataMap); Map taskRecoveryDataMap = new HashMap(); taskRecoveryDataMap.put(t1v2Id, taskRecoveryData); doReturn(taskRecoveryData).when(dagRecoveryData).getTaskRecoveryData(t1v2Id); - VertexInitializedEvent v2InitedEvent = new VertexInitializedEvent(v2Id, - "vertex2", 0L, v1InitedTime, + VertexInitializedEvent v2InitedEvent = new VertexInitializedEvent(v2Id, + "vertex2", 0L, v1InitedTime, v1NumTask, "", null, null, null); - VertexConfigurationDoneEvent v2ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v2Id, + VertexConfigurationDoneEvent v2ReconfigureDoneEvent = new VertexConfigurationDoneEvent(v2Id, 0L, v1NumTask, null, null, null, false); VertexStartedEvent v2StartedEvent = new VertexStartedEvent(v2Id, 0L, v1StartedTime); VertexRecoveryData v2RecoveryData = new VertexRecoveryData(v2InitedEvent, v2ReconfigureDoneEvent, v2StartedEvent, null, taskRecoveryDataMap, false); doReturn(v2RecoveryData).when(dagRecoveryData).getVertexRecoveryData(v2Id); - dag.handle(new DAGEventRecoverEvent(dagId, dagRecoveryData)); dispatcher.await(); - - TaskImpl task = (TaskImpl)dag.getVertex(v2Id).getTask(t1v2Id); - TaskAttemptImpl taskAttempt = (TaskAttemptImpl)task.getAttempt(ta1t1v2Id); + + TaskImpl task = (TaskImpl) dag.getVertex(v2Id).getTask(t1v2Id); + TaskAttemptImpl taskAttempt = (TaskAttemptImpl) task.getAttempt(ta1t1v2Id); assertEquals(TaskAttemptStateInternal.KILLED, taskAttempt.getInternalState()); historyEventHandler.verifyHistoryEvent(1, HistoryEventType.TASK_ATTEMPT_FINISHED); assertEquals(TaskStateInternal.RUNNING, task.getInternalState()); @@ -1371,24 +1368,24 @@ public void testTARecoverFromSucceeded_OutputCommitterRecoveryNotSupported() thr * RecoveryEvents: TaskAttemptStartedEvent -> TaskAttemptFinishedEvent (FAILED) * Recover it to FAILED */ - @Test(timeout=5000) + @Test(timeout = 5000) public void testTARecoverFromFailed() { initMockDAGRecoveryDataForTaskAttempt(); TaskAttemptStartedEvent taStartedEvent = new TaskAttemptStartedEvent( - ta1t1v1Id, "v1", ta1LaunchTime, mock(ContainerId.class), + ta1t1v1Id, "v1", ta1LaunchTime, mock(ContainerId.class), mock(NodeId.class), "", "", ""); TaskAttemptFinishedEvent taFinishedEvent = new TaskAttemptFinishedEvent( - ta1t1v1Id, "v1", ta1LaunchTime, ta1FinishedTime, + ta1t1v1Id, "v1", ta1LaunchTime, ta1FinishedTime, TaskAttemptState.FAILED, TaskFailureType.NON_FATAL, TaskAttemptTerminationCause.INPUT_READ_ERROR, "", null, null, null, 0L, null, 0L, null, null, null, null, null); TaskAttemptRecoveryData taRecoveryData = new TaskAttemptRecoveryData(taStartedEvent, taFinishedEvent); doReturn(taRecoveryData).when(dagRecoveryData).getTaskAttemptRecoveryData(ta1t1v1Id); - + dag.handle(new DAGEventRecoverEvent(dagId, dagRecoveryData)); dispatcher.await(); - - TaskImpl task = (TaskImpl)dag.getVertex(v1Id).getTask(t1v1Id); - TaskAttemptImpl taskAttempt = (TaskAttemptImpl)task.getAttempt(ta1t1v1Id); + + TaskImpl task = (TaskImpl) dag.getVertex(v1Id).getTask(t1v1Id); + TaskAttemptImpl taskAttempt = (TaskAttemptImpl) task.getAttempt(ta1t1v1Id); assertEquals(TaskAttemptStateInternal.FAILED, taskAttempt.getInternalState()); assertEquals(TaskAttemptTerminationCause.INPUT_READ_ERROR, taskAttempt.getTerminationCause()); assertEquals(TaskStateInternal.SCHEDULED, task.getInternalState()); @@ -1402,24 +1399,24 @@ public void testTARecoverFromFailed() { * RecoveryEvents: TaskAttemptStartedEvent -> TaskAttemptFinishedEvent (KILLED) * Recover it to KILLED */ - @Test(timeout=5000) + @Test(timeout = 5000) public void testTARecoverFromKilled() { initMockDAGRecoveryDataForTaskAttempt(); TaskAttemptStartedEvent taStartedEvent = new TaskAttemptStartedEvent( - ta1t1v1Id, "v1", ta1LaunchTime, mock(ContainerId.class), + ta1t1v1Id, "v1", ta1LaunchTime, mock(ContainerId.class), mock(NodeId.class), "", "", ""); TaskAttemptFinishedEvent taFinishedEvent = new TaskAttemptFinishedEvent( - ta1t1v1Id, "v1", ta1FinishedTime, ta1FinishedTime, + ta1t1v1Id, "v1", ta1FinishedTime, ta1FinishedTime, TaskAttemptState.KILLED, null, TaskAttemptTerminationCause.TERMINATED_BY_CLIENT, "", null, null, null, 0L, null, 0L, null, null, null, null, null); TaskAttemptRecoveryData taRecoveryData = new TaskAttemptRecoveryData(taStartedEvent, taFinishedEvent); doReturn(taRecoveryData).when(dagRecoveryData).getTaskAttemptRecoveryData(ta1t1v1Id); - + dag.handle(new DAGEventRecoverEvent(dagId, dagRecoveryData)); dispatcher.await(); - - TaskImpl task = (TaskImpl)dag.getVertex(v1Id).getTask(t1v1Id); - TaskAttemptImpl taskAttempt = (TaskAttemptImpl)task.getAttempt(ta1t1v1Id); + + TaskImpl task = (TaskImpl) dag.getVertex(v1Id).getTask(t1v1Id); + TaskAttemptImpl taskAttempt = (TaskAttemptImpl) task.getAttempt(ta1t1v1Id); assertEquals(TaskAttemptStateInternal.KILLED, taskAttempt.getInternalState()); assertEquals(TaskAttemptTerminationCause.TERMINATED_BY_CLIENT, taskAttempt.getTerminationCause()); historyEventHandler.verifyHistoryEvent(0, HistoryEventType.TASK_ATTEMPT_FINISHED); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java index e2df050799..bb64a7d202 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGScheduler.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -42,15 +42,15 @@ public class TestDAGScheduler { static class MockEventHandler implements EventHandler { TaskAttemptEventSchedule event; List events = Lists.newLinkedList(); + @Override public void handle(TaskAttemptEventSchedule event) { this.event = event; this.events.add(event); } } - - - @Test(timeout=5000) + + @Test(timeout = 5000) public void testDAGSchedulerNaturalOrder() { MockEventHandler mockEventHandler = new MockEventHandler(); DAG mockDag = mock(DAG.class); @@ -71,10 +71,10 @@ public void testDAGSchedulerNaturalOrder() { .thenReturn(vId1).thenReturn(vId1) .thenReturn(vId2).thenReturn(vId2) .thenReturn(vId3).thenReturn(vId3); - + DAGEventSchedulerUpdate event = new DAGEventSchedulerUpdate( - DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt); - + DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt); + DAGScheduler scheduler = new DAGSchedulerNaturalOrder(mockDag, mockEventHandler); scheduler.scheduleTaskEx(event); @@ -90,8 +90,8 @@ public void testDAGSchedulerNaturalOrder() { Assert.assertEquals(43, mockEventHandler.event.getPriorityHighLimit()); Assert.assertEquals(45, mockEventHandler.event.getPriorityLowLimit()); } - - @Test(timeout=5000) + + @Test(timeout = 5000) public void testConcurrencyLimit() { MockEventHandler mockEventHandler = new MockEventHandler(); DAG mockDag = mock(DAG.class); @@ -100,18 +100,18 @@ public void testConcurrencyLimit() { TezVertexID vId1 = TezVertexID.fromString("vertex_1436907267600_195589_1_01"); TezTaskID tId0 = TezTaskID.getInstance(vId0, 0); TezTaskID tId1 = TezTaskID.getInstance(vId1, 0); - + TaskAttempt mockAttempt; Vertex mockVertex = mock(Vertex.class); when(mockDag.getVertex((TezVertexID) any())).thenReturn(mockVertex); when(mockVertex.getDistanceFromRoot()).thenReturn(0); when(mockVertex.getVertexId()).thenReturn(vId0); - + DAGScheduler scheduler = new DAGSchedulerNaturalOrder(mockDag, mockEventHandler); scheduler.addVertexConcurrencyLimit(vId0, 0); // not effective - + // schedule beyond limit and it gets scheduled mockAttempt = mock(TaskAttempt.class); when(mockAttempt.getTaskAttemptID()).thenReturn(TezTaskAttemptID.getInstance(tId0, 0)); @@ -128,35 +128,35 @@ public void testConcurrencyLimit() { scheduler.scheduleTask(new DAGEventSchedulerUpdate( DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt)); Assert.assertEquals(3, mockEventHandler.events.size()); - + mockEventHandler.events.clear(); List mockAttempts = Lists.newArrayList(); int completed = 0; int requested = 0; int scheduled = 0; scheduler.addVertexConcurrencyLimit(vId1, 2); // effective - + // schedule beyond limit and it gets buffered mockAttempt = mock(TaskAttempt.class); mockAttempts.add(mockAttempt); when(mockAttempt.getTaskAttemptID()).thenReturn(TezTaskAttemptID.getInstance(tId1, requested++)); scheduler.scheduleTask(new DAGEventSchedulerUpdate( DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt)); - Assert.assertEquals(scheduled+1, mockEventHandler.events.size()); // scheduled + Assert.assertEquals(scheduled + 1, mockEventHandler.events.size()); // scheduled Assert.assertEquals(mockAttempts.get(scheduled).getTaskAttemptID(), mockEventHandler.events.get(scheduled).getTaskAttemptID()); // matches order scheduled++; - + mockAttempt = mock(TaskAttempt.class); mockAttempts.add(mockAttempt); when(mockAttempt.getTaskAttemptID()).thenReturn(TezTaskAttemptID.getInstance(tId1, requested++)); scheduler.scheduleTask(new DAGEventSchedulerUpdate( DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt)); - Assert.assertEquals(scheduled+1, mockEventHandler.events.size()); // scheduled + Assert.assertEquals(scheduled + 1, mockEventHandler.events.size()); // scheduled Assert.assertEquals(mockAttempts.get(scheduled).getTaskAttemptID(), mockEventHandler.events.get(scheduled).getTaskAttemptID()); // matches order scheduled++; - + mockAttempt = mock(TaskAttempt.class); mockAttempts.add(mockAttempt); when(mockAttempt.getTaskAttemptID()).thenReturn(TezTaskAttemptID.getInstance(tId1, requested++)); @@ -173,14 +173,14 @@ public void testConcurrencyLimit() { scheduler.taskCompleted(new DAGEventSchedulerUpdate( DAGEventSchedulerUpdate.UpdateType.TA_COMPLETED, mockAttempts.get(completed++))); - Assert.assertEquals(scheduled+1, mockEventHandler.events.size()); // scheduled + Assert.assertEquals(scheduled + 1, mockEventHandler.events.size()); // scheduled Assert.assertEquals(mockAttempts.get(scheduled).getTaskAttemptID(), mockEventHandler.events.get(scheduled).getTaskAttemptID()); // matches order scheduled++; scheduler.taskCompleted(new DAGEventSchedulerUpdate( DAGEventSchedulerUpdate.UpdateType.TA_COMPLETED, mockAttempts.get(completed++))); - Assert.assertEquals(scheduled+1, mockEventHandler.events.size()); // scheduled + Assert.assertEquals(scheduled + 1, mockEventHandler.events.size()); // scheduled Assert.assertEquals(mockAttempts.get(scheduled).getTaskAttemptID(), mockEventHandler.events.get(scheduled).getTaskAttemptID()); // matches order scheduled++; @@ -194,12 +194,12 @@ public void testConcurrencyLimit() { when(mockAttempt.getTaskAttemptID()).thenReturn(TezTaskAttemptID.getInstance(tId1, requested++)); scheduler.scheduleTask(new DAGEventSchedulerUpdate( DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt)); - Assert.assertEquals(scheduled+1, mockEventHandler.events.size()); // scheduled + Assert.assertEquals(scheduled + 1, mockEventHandler.events.size()); // scheduled Assert.assertEquals(mockAttempts.get(scheduled).getTaskAttemptID(), mockEventHandler.events.get(scheduled).getTaskAttemptID()); // matches order } - @Test(timeout=5000) + @Test(timeout = 5000) public void testConcurrencyLimitWithKilledNonRunningTask() { MockEventHandler mockEventHandler = new MockEventHandler(); DAG mockDag = mock(DAG.class); @@ -228,7 +228,7 @@ public void testConcurrencyLimitWithKilledNonRunningTask() { when(mockAttempt.getTaskAttemptID()).thenReturn(TezTaskAttemptID.getInstance(tId0, requested++)); scheduler.scheduleTask(new DAGEventSchedulerUpdate( DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt)); - Assert.assertEquals(scheduled+1, mockEventHandler.events.size()); // scheduled + Assert.assertEquals(scheduled + 1, mockEventHandler.events.size()); // scheduled Assert.assertEquals(mockAttempts.get(scheduled).getTaskAttemptID(), mockEventHandler.events.get(scheduled).getTaskAttemptID()); // matches order scheduled++; @@ -253,5 +253,4 @@ public void testConcurrencyLimitWithKilledNonRunningTask() { Assert.assertEquals(mockAttempts.get(0).getTaskAttemptID(), mockEventHandler.events.get(0).getTaskAttemptID()); // matches order } - } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGSchedulerNaturalOrderControlled.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGSchedulerNaturalOrderControlled.java index f79f359a7b..9abe165c8d 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGSchedulerNaturalOrderControlled.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGSchedulerNaturalOrderControlled.java @@ -96,7 +96,6 @@ public void testSimpleFlow() { verify(eventHandler, times(vertices[3].getTotalTasks() - 3)).handle(any()); reset(eventHandler); - // Schedule all tasks belonging to v4 for (int i = 0; i < vertices[4].getTotalTasks(); i++) { dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[4].getVertexId(), i, 0)); @@ -173,7 +172,6 @@ public void testSourceRequestDelayed() { reset(eventHandler); } - @SuppressWarnings("unchecked") @Test(timeout = 5000) public void testParallelismUpdated() { @@ -246,7 +244,6 @@ public void testMultipleRequestsForSameTask() { verify(eventHandler, times(vertices[0].getTotalTasks() - 1)).handle(any()); reset(eventHandler); - // Schedule all tasks belonging to v2 for (int i = 0; i < vertices[2].getTotalTasks(); i++) { dagScheduler.scheduleTaskEx(createScheduleRequest(vertices[2].getVertexId(), i, 0)); @@ -270,7 +267,6 @@ public void testMultipleRequestsForSameTask() { verify(eventHandler, times(1 + vertices[2].getTotalTasks())).handle(any()); } - // Test parallelism updated form -1 // Reduce parallelism // Different attempts scheduled for a single task. @@ -309,7 +305,6 @@ private DAG createMockDag() { doReturn(vertices[i]).when(dag).getVertex(vertexId); } - updateMockVertexWithConnections(vertices[0], createConnectionMap((Vertex[]) null), createConnectionMap(vertices[2])); updateMockVertexWithConnections(vertices[1], createConnectionMap((Vertex[]) null), @@ -324,7 +319,6 @@ private DAG createMockDag() { return dag; } - private void updateParallelismOnMockVertex(Vertex vertex, int newParallelism) { doReturn(newParallelism).when(vertex).getTotalTasks(); } @@ -374,5 +368,4 @@ private DAGEventSchedulerUpdate createScheduleRequest(TezVertexID vertexId, int TaskAttempt mockAttempt = createTaskAttempt(vertexId, taskIdInt, attemptIdInt); return new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt); } - } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestEdge.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestEdge.java index c6578ffa31..f313b932bc 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestEdge.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestEdge.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -80,7 +80,7 @@ public class TestEdge { - @Test (timeout = 5000) + @Test(timeout = 5000) public void testOneToOneEdgeManager() { EdgeManagerPluginContext mockContext = mock(EdgeManagerPluginContext.class); when(mockContext.getSourceVertexName()).thenReturn("Source"); @@ -99,7 +99,7 @@ public void testOneToOneEdgeManager() { } catch (IllegalStateException e) { Assert.assertTrue(e.getMessage().contains("1-1 source and destination task counts must match")); } - + // now make it consistent when(mockContext.getDestinationVertexNumTasks()).thenReturn(3); manager.routeDataMovementEventToDestination(event, 1, 1, destinationTaskAndInputIndices); @@ -110,7 +110,7 @@ public void testOneToOneEdgeManager() { .get(0).intValue()); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testOneToOneEdgeManagerODR() { EdgeManagerPluginContext mockContext = mock(EdgeManagerPluginContext.class); when(mockContext.getSourceVertexName()).thenReturn("Source"); @@ -161,33 +161,33 @@ public void testScatterGatherManager() { manager.getNumSourceTaskPhysicalOutputs(0); } - @SuppressWarnings({ "rawtypes" }) - @Test (timeout = 5000) + @SuppressWarnings({"rawtypes"}) + @Test(timeout = 5000) public void testCompositeEventHandling() throws TezException { EventHandler eventHandler = mock(EventHandler.class); EdgeProperty edgeProp = EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, mock(OutputDescriptor.class), mock(InputDescriptor.class)); Edge edge = new Edge(edgeProp, eventHandler, new TezConfiguration()); - + TezVertexID srcVertexID = createVertexID(1); TezVertexID destVertexID = createVertexID(2); LinkedHashMap srcTasks = mockTasks(srcVertexID, 1); LinkedHashMap destTasks = mockTasks(destVertexID, 5); - + TezTaskID srcTaskID = srcTasks.keySet().iterator().next(); - + Vertex srcVertex = mockVertex("src", srcVertexID, srcTasks); Vertex destVertex = mockVertex("dest", destVertexID, destTasks); - + edge.setSourceVertex(srcVertex); edge.setDestinationVertex(destVertex); edge.initialize(); - + TezTaskAttemptID srcTAID = createTAIDForTest(srcTaskID, 2); // Task0, Attempt 0 - + EventMetaData srcMeta = new EventMetaData(EventProducerConsumerType.OUTPUT, "consumerVertex", "producerVertex", srcTAID); - + // Verification via a CompositeEvent CompositeDataMovementEvent cdmEvent = CompositeDataMovementEvent.create(0, destTasks.size(), ByteBuffer.wrap("bytes".getBytes())); @@ -202,7 +202,7 @@ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, mock(OutputDescriptor.class // Reset the mock resetTaskMocks(destTasks.values()); - for (int i = 0 ; i < destTasks.size() ; i++) { + for (int i = 0; i < destTasks.size(); i++) { DataMovementEvent dmEvent = DataMovementEvent.create(i, ByteBuffer.wrap("bytes".getBytes())); dmEvent.setVersion(2); tezEvent = new TezEvent(dmEvent, srcMeta); @@ -240,7 +240,7 @@ private void resetTaskMocks(Collection tasks) { private LinkedHashMap mockTasks(TezVertexID vertexID, int numTasks) { LinkedHashMap tasks = new LinkedHashMap(); - for (int i = 0 ; i < numTasks ; i++) { + for (int i = 0; i < numTasks; i++) { Task task = mock(Task.class); TezTaskID taskID = TezTaskID.getInstance(vertexID, i); doReturn(taskID).when(task).getTaskID(); @@ -248,7 +248,7 @@ private LinkedHashMap mockTasks(TezVertexID vertexID, int numTa } return tasks; } - + private Vertex mockVertex(String name, TezVertexID vertexID, LinkedHashMap tasks) { Vertex vertex = mock(Vertex.class); doReturn(vertexID).when(vertex).getVertexId(); @@ -261,13 +261,13 @@ private Vertex mockVertex(String name, TezVertexID vertexID, LinkedHashMap> destinationTaskAndInputIndices) + int sourceTaskIndex, int sourceOutputIndex, + Map> destinationTaskAndInputIndices) throws Exception { } @@ -475,7 +475,7 @@ public int getNumDestinationConsumerTasks(int sourceTaskIndex) @Override public int routeInputErrorEventToSource(InputReadErrorEvent event, - int destinationTaskIndex, int destinationFailedInputIndex) + int destinationTaskIndex, int destinationFailedInputIndex) throws Exception { return emConf.sourceTaskIndex; } @@ -484,10 +484,10 @@ public int routeInputErrorEventToSource(InputReadErrorEvent event, @Test(timeout = 5000) public void testEdgeManagerPluginCtxGetVertexGroupName() throws TezException { EdgeManagerPluginDescriptor edgeManagerDescriptor = - EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName()); + EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName()); EdgeProperty edgeProp = EdgeProperty.create(edgeManagerDescriptor, - DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), - InputDescriptor.create("In")); + DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), + InputDescriptor.create("In")); Edge edge = new Edge(edgeProp, null, null); Vertex srcV = mock(Vertex.class), destV = mock(Vertex.class); @@ -501,11 +501,11 @@ public void testEdgeManagerPluginCtxGetVertexGroupName() throws TezException { String group = "group"; when(destV.getGroupInputSpecList()) - .thenReturn(Arrays.asList(new GroupInputSpec(group, Arrays.asList("v1", "v3"), null))); + .thenReturn(Arrays.asList(new GroupInputSpec(group, Arrays.asList("v1", "v3"), null))); assertNull(edge.edgeManager.getContext().getVertexGroupName()); when(destV.getGroupInputSpecList()) - .thenReturn(Arrays.asList(new GroupInputSpec(group, Arrays.asList(srcName, "v3"), null))); + .thenReturn(Arrays.asList(new GroupInputSpec(group, Arrays.asList(srcName, "v3"), null))); assertEquals(group, edge.edgeManager.getContext().getVertexGroupName()); } } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestImmediateStartVertexManager.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestImmediateStartVertexManager.java index 67a9da52b1..f5af7b76dc 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestImmediateStartVertexManager.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestImmediateStartVertexManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -47,11 +47,11 @@ import static org.mockito.Mockito.when; public class TestImmediateStartVertexManager { - - @SuppressWarnings({ "unchecked", "rawtypes" }) - @Test (timeout=5000) + + @SuppressWarnings({"unchecked", "rawtypes"}) + @Test(timeout = 5000) public void testBasic() { - HashMap mockInputVertices = + HashMap mockInputVertices = new HashMap(); final String mockSrcVertexId1 = "Vertex1"; EdgeProperty eProp1 = EdgeProperty.create( @@ -73,9 +73,9 @@ public void testBasic() { SchedulingType.SEQUENTIAL, OutputDescriptor.create("out"), InputDescriptor.create("in")); - + final String mockManagedVertexId = "Vertex4"; - + mockInputVertices.put(mockSrcVertexId1, eProp1); mockInputVertices.put(mockSrcVertexId2, eProp2); mockInputVertices.put(mockSrcVertexId3, eProp3); @@ -87,19 +87,20 @@ public void testBasic() { when(mockContext.getVertexNumTasks(mockSrcVertexId1)).thenReturn(2); when(mockContext.getVertexNumTasks(mockSrcVertexId2)).thenReturn(2); when(mockContext.getVertexNumTasks(mockSrcVertexId3)).thenReturn(2); - + final HashSet scheduledTasks = new HashSet(); doAnswer(new Answer() { public Object answer(InvocationOnMock invocation) { - Object[] args = invocation.getArguments(); - scheduledTasks.clear(); - List tasks = (List)args[0]; - for (ScheduleTaskRequest task : tasks) { - scheduledTasks.add(task.getTaskIndex()); - } - return null; - }}).when(mockContext).scheduleTasks(anyList()); - + Object[] args = invocation.getArguments(); + scheduledTasks.clear(); + List tasks = (List) args[0]; + for (ScheduleTaskRequest task : tasks) { + scheduledTasks.add(task.getTaskIndex()); + } + return null; + } + }).when(mockContext).scheduleTasks(anyList()); + List emptyCompletions = null; ImmediateStartVertexManager manager = new ImmediateStartVertexManager(mockContext); manager.initialize(); @@ -119,15 +120,15 @@ public Object answer(InvocationOnMock invocation) { final ImmediateStartVertexManager raceManager = new ImmediateStartVertexManager(mockContext); doAnswer(new Answer() { public Object answer(InvocationOnMock invocation) throws Exception { - raceManager.onVertexStateUpdated(new VertexStateUpdate((String)invocation.getArguments()[0], + raceManager.onVertexStateUpdated(new VertexStateUpdate((String) invocation.getArguments()[0], VertexState.CONFIGURED)); scheduledTasks.clear(); return null; - }}).when(mockContext).registerForVertexStateUpdates(anyString(), anySet()); + } + }).when(mockContext).registerForVertexStateUpdates(anyString(), anySet()); raceManager.initialize(); raceManager.onVertexStarted(emptyCompletions); verify(mockContext, times(2)).scheduleTasks(anyList()); Assert.assertEquals(4, scheduledTasks.size()); } - } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestRootInputVertexManager.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestRootInputVertexManager.java index 144d28d4a5..82d3bb73f3 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestRootInputVertexManager.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestRootInputVertexManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -233,16 +233,16 @@ public void testRootInputVertexManagerSlowStart() { VertexState.CONFIGURED)); Assert.assertEquals(3, manager.pendingTasks.size()); - Assert.assertEquals(numTasks*2, manager.totalNumSourceTasks); + Assert.assertEquals(numTasks * 2, manager.totalNumSourceTasks); Assert.assertEquals(0, manager.numSourceTasksCompleted); float completedTasksThreshold = 0.975f * numTasks; // Finish all tasks before exceeding the threshold - for (String mockSrcVertex : new String[] { mockSrcVertexId1, - mockSrcVertexId2 }) { + for (String mockSrcVertex : new String[]{mockSrcVertexId1, + mockSrcVertexId2}) { for (int i = 0; i < mockContext.getVertexNumTasks(mockSrcVertex); ++i) { // complete 0th tasks outside the loop manager.onSourceTaskCompleted(createTaskAttemptIdentifier( - mockSrcVertex, i+1)); + mockSrcVertex, i + 1)); if ((i + 2) >= completedTasksThreshold) { // stop before completing more than min/max source tasks break; @@ -520,18 +520,17 @@ public void testTezDrainCompletionsOnVertexStart() throws IOException { manager = createRootInputVertexManager(conf, mockContext, 0.1f, 0.1f); Assert.assertEquals(0, manager.numSourceTasksCompleted); manager.onVertexStarted(Collections.singletonList( - createTaskAttemptIdentifier(mockSrcVertexId1, 0))); + createTaskAttemptIdentifier(mockSrcVertexId1, 0))); Assert.assertEquals(1, manager.numSourceTasksCompleted); } - static RootInputVertexManager createRootInputVertexManager( Configuration conf, VertexManagerPluginContext context, Float min, - Float max) { + Float max) { if (min != null) { conf.setFloat( TEZ_ROOT_INPUT_VERTEX_MANAGER_MIN_SRC_FRACTION, - min); + min); } else { conf.unset( TEZ_ROOT_INPUT_VERTEX_MANAGER_MIN_SRC_FRACTION); @@ -539,11 +538,11 @@ static RootInputVertexManager createRootInputVertexManager( if (max != null) { conf.setFloat( TEZ_ROOT_INPUT_VERTEX_MANAGER_MAX_SRC_FRACTION, - max); + max); } else { conf.unset(TEZ_ROOT_INPUT_VERTEX_MANAGER_MAX_SRC_FRACTION); } - if(max != null || min != null) { + if (max != null || min != null) { conf.setBoolean(TEZ_ROOT_INPUT_VERTEX_MANAGER_ENABLE_SLOW_START, true); } @@ -561,16 +560,18 @@ static RootInputVertexManager createRootInputVertexManager( protected static class ScheduledTasksAnswer implements Answer { private List scheduledTasks; + public ScheduledTasksAnswer(List scheduledTasks) { this.scheduledTasks = scheduledTasks; } + @Override public Object answer(InvocationOnMock invocation) throws IOException { Object[] args = invocation.getArguments(); scheduledTasks.clear(); @SuppressWarnings("unchecked") List tasks = - (List)args[0]; + (List) args[0]; for (VertexManagerPluginContext.ScheduleTaskRequest task : tasks) { scheduledTasks.add(task.getTaskIndex()); } @@ -579,7 +580,7 @@ public Object answer(InvocationOnMock invocation) throws IOException { } public static TaskAttemptIdentifier createTaskAttemptIdentifier(String vName, - int tId) { + int tId) { VertexIdentifier mockVertex = mock(VertexIdentifier.class); when(mockVertex.getName()).thenReturn(vName); TaskIdentifier mockTask = mock(TaskIdentifier.class); @@ -590,5 +591,4 @@ public static TaskAttemptIdentifier createTaskAttemptIdentifier(String vName, when(mockAttempt.getTaskIdentifier()).thenReturn(mockTask); return mockAttempt; } - } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java index d0088bfc54..5e32e0c581 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,6 +19,7 @@ package org.apache.tez.dag.app.dag.impl; import org.apache.tez.dag.app.MockClock; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; @@ -135,7 +136,7 @@ import com.google.common.collect.Maps; -@SuppressWarnings({ "unchecked", "rawtypes" }) +@SuppressWarnings({"unchecked", "rawtypes"}) public class TestTaskAttempt { private static final Logger LOG = LoggerFactory.getLogger(TestTaskAttempt.class); @@ -146,7 +147,7 @@ public FileStatus getFileStatus(Path f) throws IOException { return new FileStatus(1, false, 1, 1, 1, f); } } - + AppContext appCtx; TezConfiguration vertexConf = new TezConfiguration(); TaskLocationHint locationHint; @@ -159,7 +160,7 @@ public FileStatus getFileStatus(Path f) throws IOException { public static void setup() { MockDNSToSwitchMapping.initializeMockRackResolver(); } - + @Before public void setupTest() { appCtx = mock(AppContext.class); @@ -219,7 +220,7 @@ public void testLocalityRequest() { fail("Second event not of type " + AMSchedulerEventTALaunchRequest.class.getName()); } - + // TODO Move the Rack request check to the client after TEZ-125 is fixed. Set requestedRacks = taImpl.taskRacks; assertEquals(1, requestedRacks.size()); @@ -307,33 +308,33 @@ public void testPriority() { when(sEvent.getPriorityHighLimit()).thenReturn(1); sta.transition(taImpl, sEvent); verify(eventHandler, times(1)).handle(arg.capture()); - AMSchedulerEventTALaunchRequest launchEvent = (AMSchedulerEventTALaunchRequest)arg.getValue(); + AMSchedulerEventTALaunchRequest launchEvent = (AMSchedulerEventTALaunchRequest) arg.getValue(); Assert.assertEquals(2, launchEvent.getPriority()); sta.transition(taImplReScheduled, sEvent); verify(eventHandler, times(2)).handle(arg.capture()); - launchEvent = (AMSchedulerEventTALaunchRequest)arg.getValue(); + launchEvent = (AMSchedulerEventTALaunchRequest) arg.getValue(); Assert.assertEquals(1, launchEvent.getPriority()); when(sEvent.getPriorityLowLimit()).thenReturn(6); when(sEvent.getPriorityHighLimit()).thenReturn(4); sta.transition(taImpl, sEvent); verify(eventHandler, times(3)).handle(arg.capture()); - launchEvent = (AMSchedulerEventTALaunchRequest)arg.getValue(); + launchEvent = (AMSchedulerEventTALaunchRequest) arg.getValue(); Assert.assertEquals(5, launchEvent.getPriority()); sta.transition(taImplReScheduled, sEvent); verify(eventHandler, times(4)).handle(arg.capture()); - launchEvent = (AMSchedulerEventTALaunchRequest)arg.getValue(); + launchEvent = (AMSchedulerEventTALaunchRequest) arg.getValue(); Assert.assertEquals(4, launchEvent.getPriority()); when(sEvent.getPriorityLowLimit()).thenReturn(5); when(sEvent.getPriorityHighLimit()).thenReturn(5); sta.transition(taImpl, sEvent); verify(eventHandler, times(5)).handle(arg.capture()); - launchEvent = (AMSchedulerEventTALaunchRequest)arg.getValue(); + launchEvent = (AMSchedulerEventTALaunchRequest) arg.getValue(); Assert.assertEquals(5, launchEvent.getPriority()); sta.transition(taImplReScheduled, sEvent); verify(eventHandler, times(6)).handle(arg.capture()); - launchEvent = (AMSchedulerEventTALaunchRequest)arg.getValue(); + launchEvent = (AMSchedulerEventTALaunchRequest) arg.getValue(); Assert.assertEquals(5, launchEvent.getPriority()); } @@ -345,9 +346,9 @@ public void testHostResolveAttempt() throws Exception { new TaskAttemptImpl.ScheduleTaskattemptTransition(); EventHandler eventHandler = mock(EventHandler.class); - String hosts[] = new String[] { "127.0.0.1", "host2", "host3" }; + String hosts[] = new String[]{"127.0.0.1", "host2", "host3"}; Set resolved = new TreeSet( - Arrays.asList(new String[]{ "host1", "host2", "host3" })); + Arrays.asList(new String[]{"host1", "host2", "host3"})); locationHint = TaskLocationHint.createTaskLocationHint( new TreeSet(Arrays.asList(hosts)), null); @@ -357,7 +358,7 @@ public void testHostResolveAttempt() throws Exception { mock(TaskCommunicatorManagerInterface.class), new Configuration(), new SystemClock(), mock(TaskHeartbeatHandler.class), appCtx, false, Resource.newInstance(1024, - 1), createFakeContainerContext(), false); + 1), createFakeContainerContext(), false); TaskAttemptImpl spyTa = spy(taImpl); when(spyTa.resolveHosts(hosts)).thenReturn( @@ -538,12 +539,11 @@ taListener, taskConf, new SystemClock(), assertEquals(2, taImpl.getDiagnostics().size()); assertEquals("Terminated", taImpl.getDiagnostics().get(1)); - + // check that original error cause is retained assertEquals(TaskAttemptTerminationCause.APPLICATION_ERROR, taImpl.getTerminationCause()); } - @Test(timeout = 5000) // Ensure ContainerTerminated is handled correctly by the TaskAttempt public void testContainerTerminatedWhileRunning() throws Exception { @@ -722,7 +722,7 @@ taListener, taskConf, new SystemClock(), assertEquals(0, taImpl.getDiagnostics().size()); assertEquals(TaskAttemptTerminationCause.UNKNOWN_ERROR, taImpl.getTerminationCause()); } - + @Test(timeout = 5000) public void testLastDataEventRecording() throws Exception { ApplicationId appId = ApplicationId.newInstance(1, 2); @@ -772,7 +772,7 @@ taListener, taskConf, new SystemClock(), taImpl.handle(new TaskAttemptEventStartedRemotely(taskAttemptID)); assertEquals("Task attempt is not in the RUNNING state", taImpl.getState(), TaskAttemptState.RUNNING); - + long ts1 = 1024; long ts2 = 2048; TezTaskAttemptID mockId1 = mock(TezTaskAttemptID.class); @@ -807,7 +807,7 @@ taListener, taskConf, new SystemClock(), assertEquals(ts2, taImpl.lastDataEvents.get(1).getTimestamp()); assertEquals(mockId2, taImpl.lastDataEvents.get(1).getTaskAttemptId()); // over-write earlier value } - + @Test(timeout = 5000) public void testFailure() throws Exception { ApplicationId appId = ApplicationId.newInstance(1, 2); @@ -865,9 +865,9 @@ taListener, taskConf, new SystemClock(), verifyEventType( arg.getAllValues().subList(0, expectedEventsAtRunning), SpeculatorEventTaskAttemptStatusUpdate.class, 1); - + taImpl.handle(new TaskAttemptEventStatusUpdate(taskAttemptID, new TaskStatusUpdateEvent(null, 0.1f, null, false))); - + taImpl.handle(new TaskAttemptEventAttemptFailed(taskAttemptID, TaskAttemptEventType.TA_FAILED, TaskFailureType.NON_FATAL, "0", TaskAttemptTerminationCause.APPLICATION_ERROR)); @@ -898,7 +898,6 @@ taListener, taskConf, new SystemClock(), arg = ArgumentCaptor.forClass(Event.class); verify(eventHandler, times(expectedEvenstAfterTerminating)).handle(arg.capture()); - Event e = verifyEventType( arg.getAllValues().subList(expectedEventsAtRunning, expectedEvenstAfterTerminating), TaskEventTAFailed.class, 1); @@ -1161,7 +1160,7 @@ taListener, taskConf, new SystemClock(), assertEquals(2, taImpl.getCounters().findCounter("group", "counter").getValue()); } - @Test (timeout = 60000L) + @Test(timeout = 60000L) public void testProgressAfterSubmit() throws Exception { ApplicationId appId = ApplicationId.newInstance(1, 2); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance( @@ -1213,14 +1212,14 @@ public void testProgressAfterSubmit() throws Exception { taImpl.handle(new TaskAttemptEventStatusUpdate( taskAttemptID, new TaskStatusUpdateEvent(null, 0.1f, null, false))); verify(eventHandler, atLeast(1)).handle(arg.capture()); - if (arg.getValue() instanceof TaskAttemptEvent) { + if (arg.getValue() instanceof TaskAttemptEvent) { taImpl.handle((TaskAttemptEvent) arg.getValue()); } Assert.assertEquals("Task Attempt's internal state should be SUBMITTED!", taImpl.getInternalState(), TaskAttemptStateInternal.SUBMITTED); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testNoProgressFail() throws Exception { ApplicationId appId = ApplicationId.newInstance(1, 2); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance( @@ -1272,7 +1271,7 @@ public void testNoProgressFail() throws Exception { assertEquals("Task attempt is not in the RUNNING state", taImpl.getState(), TaskAttemptState.RUNNING); verify(mockHeartbeatHandler).register(taskAttemptID); - + when(mockClock.getTime()).thenReturn(100l); taImpl.handle(new TaskAttemptEventStatusUpdate( taskAttemptID, new TaskStatusUpdateEvent(null, 0.1f, null, true))); @@ -1318,7 +1317,7 @@ public void testEventSerializingHash() throws Exception { TezTaskAttemptID taID11 = TezTaskAttemptID.getInstance(taskID1, 0); TezTaskAttemptID taID12 = TezTaskAttemptID.getInstance(taskID1, 1); TezTaskAttemptID taID21 = TezTaskAttemptID.getInstance(taskID2, 1); - + TaskAttemptEvent taEventFail11 = new TaskAttemptEvent(taID11, TaskAttemptEventType.TA_FAILED); TaskAttemptEvent taEventKill11 = new TaskAttemptEvent(taID11, TaskAttemptEventType.TA_KILL_REQUEST); TaskAttemptEvent taEventKill12 = new TaskAttemptEvent(taID12, TaskAttemptEventType.TA_KILL_REQUEST); @@ -1326,7 +1325,7 @@ public void testEventSerializingHash() throws Exception { TaskEvent tEventKill1 = new TaskEvent(taskID1, TaskEventType.T_ATTEMPT_KILLED); TaskEvent tEventFail1 = new TaskEvent(taskID1, TaskEventType.T_ATTEMPT_FAILED); TaskEvent tEventFail2 = new TaskEvent(taskID2, TaskEventType.T_ATTEMPT_FAILED); - + // all of them should have the same value assertEquals(taEventFail11.getSerializingHash(), taEventKill11.getSerializingHash()); assertEquals(taEventKill11.getSerializingHash(), taEventKill12.getSerializingHash()); @@ -1406,7 +1405,6 @@ taListener, taskConf, new SystemClock(), arg = ArgumentCaptor.forClass(Event.class); verify(eventHandler, times(expectedEvenstAfterTerminating)).handle(arg.capture()); - Event e = verifyEventType( arg.getAllValues().subList(expectedEventsAtStarting, expectedEvenstAfterTerminating), TaskEventTASucceeded.class, 1); @@ -1476,10 +1474,10 @@ taListener, taskConf, new SystemClock(), verifyEventType( arg.getAllValues().subList(0, expectedEventsAtRunning), SpeculatorEventTaskAttemptStatusUpdate.class, 1); - - taImpl.handle(new TaskAttemptEventStatusUpdate(taskAttemptID, + + taImpl.handle(new TaskAttemptEventStatusUpdate(taskAttemptID, new TaskStatusUpdateEvent(null, 0.1f, null, false))); - + taImpl.handle(new TaskAttemptEvent(taskAttemptID, TaskAttemptEventType.TA_DONE)); assertEquals("Task attempt is not in the SUCCEEDED state", taImpl.getState(), @@ -1505,7 +1503,7 @@ taListener, taskConf, new SystemClock(), arg.getAllValues().subList(expectedEventsAtRunning, expectedEvenstAfterTerminating), SpeculatorEventTaskAttemptStatusUpdate.class, 2); } - + @Test(timeout = 5000) // Ensure Container Preemption race with task completion is handled correctly by // the TaskAttempt @@ -1705,7 +1703,7 @@ taListener, taskConf, new SystemClock(), "InternalError occurred trying to handle TA_TEZ_EVENT_UPDATE in KILLED state", eventHandler.internalError); } - + @Test(timeout = 5000) // Ensure node failure on Successful Leaf tasks do not cause them to be marked as KILLED public void testNodeFailedLeafVertex() throws Exception { @@ -1784,7 +1782,7 @@ taListener, taskConf, new SystemClock(), expectedEvenstAfterTerminating), DAGEventCounterUpdate.class, 1); // Send out a Node Failure. - taImpl.handle(new TaskAttemptEventNodeFailed(taskAttemptID, "NodeDecomissioned", + taImpl.handle(new TaskAttemptEventNodeFailed(taskAttemptID, "NodeDecomissioned", TaskAttemptTerminationCause.NODE_FAILED)); // Verify no additional events @@ -1866,7 +1864,7 @@ taListener, taskConf, new SystemClock(), verify(eventHandler, times(expectedEventsTillSucceeded)).handle(arg.capture()); verify(mockHistHandler, times(2)).handle(histArg.capture()); // start and finish DAGHistoryEvent histEvent = histArg.getValue(); - TaskAttemptFinishedEvent finishEvent = (TaskAttemptFinishedEvent)histEvent.getHistoryEvent(); + TaskAttemptFinishedEvent finishEvent = (TaskAttemptFinishedEvent) histEvent.getHistoryEvent(); long finishTime = finishEvent.getFinishTime(); verifyEventType(arg.getAllValues(), TaskEventTAUpdate.class, 2); @@ -1885,7 +1883,7 @@ taListener, taskConf, new SystemClock(), when(appCtx.getCurrentDAG()).thenReturn(mockDAG); TezEvent tzEvent = new TezEvent(mockReEvent, mockMeta); taImpl.handle(new TaskAttemptEventOutputFailed(taskAttemptID, tzEvent, 11)); - + // failure threshold not met. state is SUCCEEDED assertEquals("Task attempt is not in succeeded state", taImpl.getState(), TaskAttemptState.SUCCEEDED); @@ -1909,14 +1907,14 @@ taListener, taskConf, new SystemClock(), when(destVertex.getRunningTasks()).thenReturn(11); when(mockDAG.getVertex(destVertexID)).thenReturn(destVertex); taImpl.handle(new TaskAttemptEventOutputFailed(taskAttemptID, tzEvent, 11)); - + assertEquals("Task attempt is not in FAILED state", TaskAttemptState.FAILED, taImpl.getState()); assertEquals(TaskAttemptTerminationCause.OUTPUT_LOST, taImpl.getTerminationCause()); // verify unregister is not invoked again verify(mockHeartbeatHandler, times(1)).unregister(taskAttemptID); verify(mockHistHandler, times(3)).handle(histArg.capture()); histEvent = histArg.getValue(); - finishEvent = (TaskAttemptFinishedEvent)histEvent.getHistoryEvent(); + finishEvent = (TaskAttemptFinishedEvent) histEvent.getHistoryEvent(); assertEquals(TaskFailureType.NON_FATAL, finishEvent.getTaskFailureType()); long newFinishTime = finishEvent.getFinishTime(); Assert.assertEquals(finishTime, newFinishTime); @@ -1978,7 +1976,7 @@ taListener, taskConf, new SystemClock(), // verify unregister is not invoked again verify(mockHeartbeatHandler, times(1)).unregister(taskAttemptID2); - Clock mockClock = mock(Clock.class); + Clock mockClock = mock(Clock.class); int readErrorTimespanSec = 1; newVertexConf = new Configuration(vertexConf); @@ -2095,7 +2093,7 @@ taListener, taskConf, new SystemClock(), verify(eventHandler, times(expectedEventsTillSucceeded)).handle(arg.capture()); verify(mockHistHandler, times(2)).handle(histArg.capture()); // start and finish DAGHistoryEvent histEvent = histArg.getValue(); - TaskAttemptFinishedEvent finishEvent = (TaskAttemptFinishedEvent)histEvent.getHistoryEvent(); + TaskAttemptFinishedEvent finishEvent = (TaskAttemptFinishedEvent) histEvent.getHistoryEvent(); long finishTime = finishEvent.getFinishTime(); verifyEventType(arg.getAllValues(), TaskEventTAUpdate.class, 2); @@ -2181,7 +2179,7 @@ public void testMapTaskIsBlamedImmediatelyOnLocalFetchFailure() throws ServicePl } private void testMapTaskFailingForFetchFailureType(boolean isLocalFetch, - boolean isDiskErrorAtSource, TaskAttemptStateInternal expectedState) { + boolean isDiskErrorAtSource, TaskAttemptStateInternal expectedState) { EventHandler eventHandler = mock(EventHandler.class); TezTaskID taskID = TezTaskID.getInstance(TezVertexID.getInstance(TezDAGID.getInstance("1", 1, 1), 1), 1); @@ -2220,7 +2218,7 @@ public void testMapTaskIsBlamedByDownstreamAttemptsFromDifferentHosts() { TezTaskID taskID = TezTaskID.getInstance(TezVertexID.getInstance(TezDAGID.getInstance("1", 1, 1), 1), 1); TaskAttemptImpl sourceAttempt = new MockTaskAttemptImpl(taskID, 1, eventHandler, null, new Configuration(), SystemClock.getInstance(), mock(TaskHeartbeatHandler.class), appCtx, false, null, null, false) - .setNodeId(NodeId.newInstance("somehost", 0)); + .setNodeId(NodeId.newInstance("somehost", 0)); // input read error events from 2 different hosts InputReadErrorEvent inputReadErrorEvent1 = @@ -2266,7 +2264,7 @@ public void testMapTaskIsBlamedByDownstreamAttemptsFromDifferentHosts() { } private Event verifyEventType(List events, - Class eventClass, int expectedOccurences) { + Class eventClass, int expectedOccurences) { int count = 0; Event ret = null; for (Event e : events) { @@ -2296,15 +2294,16 @@ public void handle(Event event) { } private class MockTaskAttemptImpl extends TaskAttemptImpl { - + public int taskAttemptStartedEventLogged = 0; public int taskAttemptFinishedEventLogged = 0; + public MockTaskAttemptImpl(TezTaskID taskId, int attemptNumber, - EventHandler eventHandler, TaskCommunicatorManagerInterface tal, - Configuration conf, Clock clock, - TaskHeartbeatHandler taskHeartbeatHandler, AppContext appContext, - boolean isRescheduled, - Resource resource, ContainerContext containerContext, boolean leafVertex) { + EventHandler eventHandler, TaskCommunicatorManagerInterface tal, + Configuration conf, Clock clock, + TaskHeartbeatHandler taskHeartbeatHandler, AppContext appContext, + boolean isRescheduled, + Resource resource, ContainerContext containerContext, boolean leafVertex) { super(TezBuilderUtils.newTaskAttemptId(taskId, attemptNumber), eventHandler, tal, conf, clock, taskHeartbeatHandler, appContext, @@ -2343,7 +2342,7 @@ protected void logJobHistoryAttemptUnsuccesfulCompletion( taskAttemptFinishedEventLogged++; super.logJobHistoryAttemptUnsuccesfulCompletion(state, taskFailureType); } - + @Override protected void sendInputFailedToConsumers() { inputFailedReported = true; diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java index 3bf59977e5..ac45c1fe52 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -136,15 +136,17 @@ public class TestTaskImpl { private MockTaskImpl mockTask; private TaskSpec mockTaskSpec; private Vertex mockVertex; - + @SuppressWarnings("rawtypes") class TestEventHandler implements EventHandler { List events = new ArrayList(); + @Override public void handle(Event event) { events.add(event); } } + private TestEventHandler eventHandler; @Before @@ -193,7 +195,7 @@ public void setup() { mockTaskSpec = mock(TaskSpec.class); mockVertex = mock(Vertex.class); ServicePluginInfo servicePluginInfo = new ServicePluginInfo() - .setContainerLauncherName(TezConstants.getTezYarnServicePluginName()); + .setContainerLauncherName(TezConstants.getTezYarnServicePluginName()); when(mockVertex.getServicePluginInfo()).thenReturn(servicePluginInfo); when(mockVertex.getVertexConfig()).thenReturn(new VertexImpl.VertexConfigImpl(conf)); } @@ -282,7 +284,7 @@ private void updateAttemptProgress(MockTaskAttemptImpl attempt, float p) { } private void updateAttemptState(MockTaskAttemptImpl attempt, - TaskAttemptState s) { + TaskAttemptState s) { attempt.setState(s); } @@ -291,7 +293,7 @@ private void killRunningTaskAttempt(TezTaskAttemptID attemptId, TaskState stateT } private void killRunningTaskAttempt(TezTaskAttemptID attemptId, TaskState stateToVerify, - int killedCountToVerify) { + int killedCountToVerify) { mockTask.handle(createTaskTAKilledEvent(attemptId)); assertTaskState(stateToVerify); verify(mockTask.getVertex(), times(killedCountToVerify)).incrementKilledTaskAttemptCount(); @@ -377,7 +379,7 @@ public void testKillScheduledTask() { scheduleTaskAttempt(taskId); killTask(taskId); } - + /** * {@link TaskState#RUNNING}->{@link TaskState#KILLED} */ @@ -483,7 +485,7 @@ public void testKillRunningTaskButAttemptSucceeds() { mockTask.handle(createTaskTASucceededEvent(mockTask.getLastAttempt().getTaskAttemptID())); assertEquals(TaskStateInternal.KILLED, mockTask.getInternalState()); } - + /** * {@link TaskState#RUNNING}->{@link TaskState#KILLED} */ @@ -576,7 +578,6 @@ public void testKilledAttemptAtTaskKilled() { // of killed attempts in killed state. mockTask.handle(createTaskTAKilledEvent(mockTask.getLastAttempt().getTaskAttemptID())); assertEquals(TaskStateInternal.KILLED, mockTask.getInternalState()); - } /** @@ -595,11 +596,8 @@ public void testKilledAttemptAtTaskFailed() { // Send kill for an attempt mockTask.handle(createTaskTAKilledEvent(mockTask.getLastAttempt().getTaskAttemptID())); assertEquals(TaskStateInternal.FAILED, mockTask.getInternalState()); - } - - @Test(timeout = 5000) public void testFetchedEventsModifyUnderlyingList() { // Tests to ensure that adding an event to a task, does not affect the @@ -693,7 +691,6 @@ public void testFailureDuringTaskAttemptCommit() { assertTaskSucceededState(); } - @Test(timeout = 5000) public void testEventBacklogDuringTaskAttemptCommit() { @@ -715,7 +712,6 @@ public void testEventBacklogDuringTaskAttemptCommit() { assertTaskSucceededState(); } - @Test(timeout = 5000) public void testChangeCommitTaskAttempt() { TezTaskID taskId = getNewTaskID(); @@ -723,17 +719,17 @@ public void testChangeCommitTaskAttempt() { launchTaskAttempt(mockTask.getLastAttempt().getTaskAttemptID()); updateAttemptState(mockTask.getLastAttempt(), TaskAttemptState.RUNNING); TezTaskAttemptID lastTAId = mockTask.getLastAttempt().getTaskAttemptID(); - + // Add a speculative task attempt that succeeds mockTask.handle(createTaskTAAddSpecAttempt(mockTask.getLastAttempt().getTaskAttemptID())); launchTaskAttempt(mockTask.getLastAttempt().getTaskAttemptID()); updateAttemptState(mockTask.getLastAttempt(), TaskAttemptState.RUNNING); - + assertEquals(2, mockTask.getAttemptList().size()); - + // previous running attempt should be the casual TA of this speculative attempt Assert.assertEquals(lastTAId, mockTask.getLastAttempt().getSchedulingCausalTA()); - + assertTrue("Second attempt should commit", mockTask.canCommit(mockTask.getAttemptList().get(1).getTaskAttemptID())); assertFalse("First attempt should not commit", @@ -745,7 +741,7 @@ public void testChangeCommitTaskAttempt() { failRunningTaskAttempt(mockTask.getLastAttempt().getTaskAttemptID()); assertEquals(2, mockTask.getAttemptList().size()); - + assertFalse("Second attempt should not commit", mockTask.canCommit(mockTask.getAttemptList().get(1).getTaskAttemptID())); assertTrue("First attempt should commit", @@ -756,7 +752,7 @@ public void testChangeCommitTaskAttempt() { assertTaskSucceededState(); } - + @SuppressWarnings("rawtypes") @Test(timeout = 5000) public void testTaskSucceedAndRetroActiveFailure() { @@ -777,7 +773,7 @@ public void testTaskSucceedAndRetroActiveFailure() { DAGHistoryEvent dagHistoryEvent = argumentCaptor.getValue(); HistoryEvent historyEvent = dagHistoryEvent.getHistoryEvent(); assertTrue(historyEvent instanceof TaskFinishedEvent); - TaskFinishedEvent taskFinishedEvent = (TaskFinishedEvent)historyEvent; + TaskFinishedEvent taskFinishedEvent = (TaskFinishedEvent) historyEvent; assertEquals(taskFinishedEvent.getFinishTime(), mockTask.getFinishTime()); eventHandler.events.clear(); @@ -786,7 +782,7 @@ public void testTaskSucceedAndRetroActiveFailure() { TezEvent mockTezEvent = mock(TezEvent.class); EventMetaData meta = new EventMetaData(EventProducerConsumerType.INPUT, "Vertex", "Edge", mockDestId); when(mockTezEvent.getSourceInfo()).thenReturn(meta); - TaskAttemptEventOutputFailed outputFailedEvent = + TaskAttemptEventOutputFailed outputFailedEvent = new TaskAttemptEventOutputFailed(mockDestId, mockTezEvent, 1); mockTask.handle( createTaskTAFailedEvent(mockTask.getLastAttempt().getTaskAttemptID(), TaskFailureType.NON_FATAL, @@ -796,9 +792,9 @@ public void testTaskSucceedAndRetroActiveFailure() { assertTaskScheduledState(); Event event = eventHandler.events.get(0); Assert.assertEquals(AMNodeEventType.N_TA_ENDED, event.getType()); - event = eventHandler.events.get(eventHandler.events.size()-1); + event = eventHandler.events.get(eventHandler.events.size() - 1); Assert.assertEquals(VertexEventType.V_TASK_RESCHEDULED, event.getType()); - + // report of output read error should be the causal TA List attempts = mockTask.getAttemptList(); Assert.assertEquals(2, attempts.size()); @@ -832,7 +828,7 @@ public void testTaskSucceedAndRetroActiveKilled() { } @Test(timeout = 5000) - public void testDiagnostics_KillNew(){ + public void testDiagnostics_KillNew() { TezTaskID taskId = getNewTaskID(); mockTask.handle(new TaskEventTermination(taskId, TaskAttemptTerminationCause.TERMINATED_BY_CLIENT, null)); assertEquals(1, mockTask.getDiagnostics().size()); @@ -840,9 +836,9 @@ public void testDiagnostics_KillNew(){ assertEquals(0, mockTask.taskStartedEventLogged); assertEquals(1, mockTask.taskFinishedEventLogged); } - + @Test(timeout = 5000) - public void testDiagnostics_Kill(){ + public void testDiagnostics_Kill() { TezTaskID taskId = getNewTaskID(); scheduleTaskAttempt(taskId); mockTask.handle(new TaskEventTermination(taskId, TaskAttemptTerminationCause.TERMINATED_AT_SHUTDOWN, null)); @@ -985,7 +981,7 @@ public void testKilledAttemptUpdatesDAGScheduler() { verify(mockTask.stateChangeNotifier).taskSucceeded(any(), eq(taskId), eq(firstAttempt.getTaskAttemptID().getId())); @SuppressWarnings("rawtypes") - Event event = eventHandler.events.get(eventHandler.events.size()-1); + Event event = eventHandler.events.get(eventHandler.events.size() - 1); assertEquals(TaskAttemptEventType.TA_KILL_REQUEST, event.getType()); assertEquals(specAttempt.getTaskAttemptID(), ((TaskAttemptEventKillRequest) event).getTaskAttemptID()); @@ -1022,7 +1018,7 @@ public void testSpeculatedThenRetroactiveFailure() { verify(mockTask.stateChangeNotifier).taskSucceeded(any(), eq(taskId), eq(firstAttempt.getTaskAttemptID().getId())); @SuppressWarnings("rawtypes") - Event event = eventHandler.events.get(eventHandler.events.size()-1); + Event event = eventHandler.events.get(eventHandler.events.size() - 1); assertEquals(TaskAttemptEventType.TA_KILL_REQUEST, event.getType()); assertEquals(specAttempt.getTaskAttemptID(), ((TaskAttemptEventKillRequest) event).getTaskAttemptID()); @@ -1043,7 +1039,7 @@ public void testSpeculatedThenRetroactiveFailure() { // The task should still be in the scheduled state assertTaskScheduledState(); - event = eventHandler.events.get(eventHandler.events.size()-1); + event = eventHandler.events.get(eventHandler.events.size() - 1); Assert.assertEquals(VertexEventType.V_TASK_RESCHEDULED, event.getType()); // There should be a new attempt, and report of output read error @@ -1114,16 +1110,16 @@ public void testSucceededAttemptStatusWithRetroActiveFailures() throws Interrupt new TaskAttemptEventStartedRemotely(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()))); secondMockTaskAttempt.handle( new TaskAttemptEvent(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), - TaskAttemptEventType.TA_DONE)); + TaskAttemptEventType.TA_DONE)); firstMockTaskAttempt.handle( new TaskAttemptEvent(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), - TaskAttemptEventType.TA_DONE)); + TaskAttemptEventType.TA_DONE)); mockTask.handle(new TaskEventTASucceeded(secondMockTaskAttempt.getTaskAttemptID())); mockTask.handle(new TaskEventTASucceeded(firstMockTaskAttempt.getTaskAttemptID())); assertTrue("Attempts should have succeeded!", firstMockTaskAttempt.getInternalState() == TaskAttemptStateInternal.SUCCEEDED - && secondMockTaskAttempt.getInternalState() == TaskAttemptStateInternal.SUCCEEDED); + && secondMockTaskAttempt.getInternalState() == TaskAttemptStateInternal.SUCCEEDED); assertEquals("Task should have no uncompleted attempts!", 0, mockTask.getUncompletedAttemptsCount()); assertTrue("Task should have Succeeded!", mockTask.getState() == TaskState.SUCCEEDED); //Failing the attempt that finished after the task was marked succeeded, should not schedule another attempt @@ -1142,9 +1138,9 @@ public void testFailedAttemptStatus() throws InterruptedException { Vertex vertex = mock(Vertex.class); doReturn(new VertexImpl.VertexConfigImpl(newConf)).when(vertex).getVertexConfig(); mockTask = new MockTaskImpl(vertexId, partition, - eventHandler, conf, taskCommunicatorManagerInterface, clock, - taskHeartbeatHandler, appContext, leafVertex, - taskResource, containerContext, vertex); + eventHandler, conf, taskCommunicatorManagerInterface, clock, + taskHeartbeatHandler, appContext, leafVertex, + taskResource, containerContext, vertex); TezTaskID taskId = getNewTaskID(); scheduleTaskAttempt(taskId); MockTaskAttemptImpl firstMockTaskAttempt = mockTask.getAttemptList().get(0); @@ -1168,12 +1164,12 @@ public void testFailedAttemptStatus() throws InterruptedException { new TaskAttemptEventStartedRemotely(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()))); secondMockTaskAttempt.handle( new TaskAttemptEventAttemptFailed(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), - TaskAttemptEventType.TA_FAILED,TaskFailureType.NON_FATAL, "test", - TaskAttemptTerminationCause.NO_PROGRESS)); + TaskAttemptEventType.TA_FAILED, TaskFailureType.NON_FATAL, "test", + TaskAttemptTerminationCause.NO_PROGRESS)); firstMockTaskAttempt.handle( new TaskAttemptEventAttemptFailed(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), - TaskAttemptEventType.TA_FAILED, TaskFailureType.NON_FATAL, "test", - TaskAttemptTerminationCause.NO_PROGRESS)); + TaskAttemptEventType.TA_FAILED, TaskFailureType.NON_FATAL, "test", + TaskAttemptTerminationCause.NO_PROGRESS)); firstMockTaskAttempt.handle(new TaskAttemptEventContainerTerminated(mockContainerId, firstMockTaskAttempt.getTaskAttemptID(), "test", TaskAttemptTerminationCause.NO_PROGRESS)); @@ -1185,13 +1181,13 @@ public void testFailedAttemptStatus() throws InterruptedException { mock(TaskAttemptEvent.class))); assertTrue("Attempts should have failed!", firstMockTaskAttempt.getInternalState() == TaskAttemptStateInternal.FAILED - && secondMockTaskAttempt.getInternalState() == TaskAttemptStateInternal.FAILED); + && secondMockTaskAttempt.getInternalState() == TaskAttemptStateInternal.FAILED); assertEquals("Task should have no uncompleted attempts!", 0, mockTask.getUncompletedAttemptsCount()); assertTrue("Task should have failed!", mockTask.getState() == TaskState.FAILED); } @SuppressWarnings("rawtypes") - @Test (timeout = 10000L) + @Test(timeout = 10000L) public void testSucceededLeafTaskWithRetroFailures() throws InterruptedException { Configuration newConf = new Configuration(conf); newConf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 1); @@ -1261,7 +1257,7 @@ private void failAttempt(MockTaskAttemptImpl taskAttempt, int index, int expecte expectedIncompleteAttempts, mockTask.getUncompletedAttemptsCount()); } - @Test (timeout = 30000) + @Test(timeout = 30000) public void testFailedTaskTransitionWithLaunchedAttempt() throws InterruptedException { Configuration newConf = new Configuration(conf); newConf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 1); @@ -1294,7 +1290,7 @@ public void testFailedTaskTransitionWithLaunchedAttempt() throws InterruptedExce new TaskAttemptEventStartedRemotely(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()))); secondMockTaskAttempt.handle( new TaskAttemptEventAttemptFailed(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), - TaskAttemptEventType.TA_FAILED,TaskFailureType.NON_FATAL, "test", + TaskAttemptEventType.TA_FAILED, TaskFailureType.NON_FATAL, "test", TaskAttemptTerminationCause.NO_PROGRESS)); firstMockTaskAttempt.handle( new TaskAttemptEventAttemptFailed(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), @@ -1319,7 +1315,7 @@ public void testFailedTaskTransitionWithLaunchedAttempt() throws InterruptedExce mockTask.handle(createTaskTALauncherEvent(thirdMockTaskAttempt.getTaskAttemptID())); } - @Test (timeout = 30000) + @Test(timeout = 30000) public void testKilledTaskTransitionWithLaunchedAttempt() throws InterruptedException { TezTaskID taskId = getNewTaskID(); scheduleTaskAttempt(taskId); @@ -1345,17 +1341,17 @@ public void testKilledTaskTransitionWithLaunchedAttempt() throws InterruptedExce mockTask.handle(new TaskEventTermination(mockTask.getTaskID(), TaskAttemptTerminationCause.FRAMEWORK_ERROR, "test")); secondMockTaskAttempt.handle( - new TaskAttemptEventAttemptKilled(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()),"test", + new TaskAttemptEventAttemptKilled(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), "test", TaskAttemptTerminationCause.FRAMEWORK_ERROR)); mockTask.handle(new TaskEventTAKilled(secondMockTaskAttempt.getTaskAttemptID(), new TaskAttemptEvent(secondMockTaskAttempt.getTaskAttemptID(), TaskAttemptEventType.TA_KILLED))); firstMockTaskAttempt.handle( - new TaskAttemptEventAttemptKilled(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()),"test", + new TaskAttemptEventAttemptKilled(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), "test", TaskAttemptTerminationCause.FRAMEWORK_ERROR)); mockTask.handle(new TaskEventTAKilled(firstMockTaskAttempt.getTaskAttemptID(), new TaskAttemptEvent(firstMockTaskAttempt.getTaskAttemptID(), TaskAttemptEventType.TA_KILLED))); firstMockTaskAttempt.handle( - new TaskAttemptEventAttemptKilled(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()),"test", + new TaskAttemptEventAttemptKilled(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), "test", TaskAttemptTerminationCause.FRAMEWORK_ERROR)); assertEquals("Task should have been killed!", mockTask.getInternalState(), TaskStateInternal.KILLED); mockTask.handle(createTaskTAAddSpecAttempt(mockTask.getLastAttempt().getTaskAttemptID())); @@ -1370,7 +1366,6 @@ public void testKilledTaskTransitionWithLaunchedAttempt() throws InterruptedExce // TODO Add test to validate the correct commit attempt. - /* Verifies that the specified event types, exist. Does not ensure they are the only ones, however */ private void verifyOutgoingEvents(List events, Enum... expectedTypes) { @@ -1404,11 +1399,11 @@ private class MockTaskImpl extends TaskImpl { private Vertex vertex; public MockTaskImpl(TezVertexID vertexId, int partition, - EventHandler eventHandler, Configuration conf, - TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, Clock clock, - TaskHeartbeatHandler thh, AppContext appContext, boolean leafVertex, - Resource resource, - ContainerContext containerContext, Vertex vertex) { + EventHandler eventHandler, Configuration conf, + TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, Clock clock, + TaskHeartbeatHandler thh, AppContext appContext, boolean leafVertex, + Resource resource, + ContainerContext containerContext, Vertex vertex) { super(vertexId, partition, eventHandler, conf, taskCommunicatorManagerInterface, clock, thh, appContext, leafVertex, resource, containerContext, mock(StateChangeNotifier.class), vertex); @@ -1439,7 +1434,7 @@ MockTaskAttemptImpl getLastAttempt() { List getAttemptList() { return taskAttempts; } - + @Override public Vertex getVertex() { return vertex; @@ -1466,10 +1461,10 @@ public class MockTaskAttemptImpl extends TaskAttemptImpl { private TaskAttemptState state = TaskAttemptState.NEW; public MockTaskAttemptImpl(TezTaskAttemptID attemptId, - EventHandler eventHandler, TaskCommunicatorManagerInterface tal, Configuration conf, - Clock clock, TaskHeartbeatHandler thh, AppContext appContext, - boolean isRescheduled, - Resource resource, ContainerContext containerContext, TezTaskAttemptID schedCausalTA) { + EventHandler eventHandler, TaskCommunicatorManagerInterface tal, Configuration conf, + Clock clock, TaskHeartbeatHandler thh, AppContext appContext, + boolean isRescheduled, + Resource resource, ContainerContext containerContext, TezTaskAttemptID schedCausalTA) { super(attemptId, eventHandler, tal, conf, clock, thh, appContext, isRescheduled, resource, containerContext, false, mockTask, locationHint, mockTaskSpec, schedCausalTA); @@ -1497,12 +1492,12 @@ public void setState(TaskAttemptState state) { public TaskAttemptState getState() { return state; } - + @Override public TaskAttemptState getStateNoLock() { return state; } - + @Override public ContainerId getAssignedContainerID() { return mockContainerId; @@ -1515,7 +1510,7 @@ public NodeId getNodeId() { } public class ServiceBusyEvent extends TezAbstractEvent - implements TaskAttemptEventTerminationCauseEvent { + implements TaskAttemptEventTerminationCauseEvent { public ServiceBusyEvent() { super(TaskAttemptEventType.TA_KILLED); } @@ -1526,4 +1521,3 @@ public TaskAttemptTerminationCause getTerminationCause() { } } } - diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java index 10ecfe503c..34f3938c79 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,6 +29,7 @@ import org.apache.tez.dag.app.launcher.TezContainerLauncherImpl; import org.apache.tez.dag.app.rm.container.AMContainer; import org.apache.tez.serviceplugins.api.ContainerLauncherDescriptor; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -257,7 +258,7 @@ public class TestVertexImpl { private boolean useCustomInitializer = false; private InputInitializer customInitializer = null; - + private TezDAGID dagId; private ApplicationAttemptId appAttemptId; private DAGPlan dagPlan; @@ -360,14 +361,14 @@ public CountingOutputCommitterConfig() { } public CountingOutputCommitterConfig(boolean throwError, - boolean throwErrorOnAbort, boolean throwRuntimeException) { + boolean throwErrorOnAbort, boolean throwRuntimeException) { this.throwError = throwError; this.throwErrorOnAbort = throwErrorOnAbort; this.throwRuntimeException = throwRuntimeException; } public CountingOutputCommitterConfig(UserPayload payload) throws IOException { - DataInputByteBuffer in = new DataInputByteBuffer(); + DataInputByteBuffer in = new DataInputByteBuffer(); in.reset(payload.getPayload()); this.readFields(in); } @@ -392,25 +393,24 @@ public byte[] toUserPayload() throws IOException { write(out); return bos.toByteArray(); } - } - } - + private class TaskAttemptEventDispatcher implements EventHandler { @SuppressWarnings("unchecked") @Override public void handle(TaskAttemptEvent event) { VertexImpl vertex = vertexIdMap.get( - event.getVertexID()); + event.getVertexID()); Task task = vertex.getTask(event.getTaskAttemptID().getTaskID()); - ((EventHandler)task.getAttempt( + ((EventHandler) task.getAttempt( event.getTaskAttemptID())).handle(event); } } private class TaskEventDispatcher implements EventHandler { List events = Lists.newArrayList(); + @SuppressWarnings("unchecked") @Override public void handle(TaskEvent event) { @@ -418,7 +418,7 @@ public void handle(TaskEvent event) { VertexImpl vertex = vertexIdMap.get(event.getVertexID()); Task task = vertex.getTask(event.getTaskID()); if (task != null) { - ((EventHandler)task).handle(event); + ((EventHandler) task).handle(event); } else { LOG.warn("Task null for vertex: " + vertex.getName() + " taskId: " + event.getTaskID() + ". Please check if this is important for the test"); @@ -440,7 +440,7 @@ public void handle(DAGEvent event) { } } - private class AMSchedulerEventDispatcher implements EventHandler { + private class AMSchedulerEventDispatcher implements EventHandler { List events = new ArrayList<>(); public void handle(AMSchedulerEvent event) { @@ -464,24 +464,24 @@ private DAGPlan createInvalidDAGPlan() { .setName("testverteximplinvalid") .addVertex( VertexPlan.newBuilder() - .setName("vertex1") - .setType(PlanVertexType.NORMAL) - .addTaskLocationHint( - PlanTaskLocationHint.newBuilder() - .addHost("host1") - .addRack("rack1") + .setName("vertex1") + .setType(PlanVertexType.NORMAL) + .addTaskLocationHint( + PlanTaskLocationHint.newBuilder() + .addHost("host1") + .addRack("rack1") + .build() + ) + .setTaskConfig( + PlanTaskConfiguration.newBuilder() + .setNumTasks(0) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("x1.y1") + .build() + ) .build() - ) - .setTaskConfig( - PlanTaskConfiguration.newBuilder() - .setNumTasks(0) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("x1.y1") - .build() - ) - .build() ) .build(); return dag; @@ -806,7 +806,7 @@ private DAGPlan createDAGPlanWithNonExistVertexManager() { .setClassName("non-exist-vertexmanager")) .build() ).build(); - return dag; + return dag; } private DAGPlan createDAGPlanWithMixedEdges() { @@ -849,7 +849,7 @@ private DAGPlan createDAGPlanWithMixedEdges() { EdgeProperty.create(DataMovementType.ONE_TO_ONE, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("out.class"), InputDescriptor.create("out.class")))); - + return dag.createDag(conf, null, null, null, true); } @@ -1040,7 +1040,7 @@ private DAGPlan createDAGPlanWithInputInitializer(String initializerClassName) { .build() ) .build() - ) + ) .setTaskConfig( PlanTaskConfiguration.newBuilder() .setNumTasks(-1) @@ -1054,40 +1054,40 @@ private DAGPlan createDAGPlanWithInputInitializer(String initializerClassName) { .setClassName(RootInputSpecUpdaterVertexManager.class.getName()) .setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder() .setUserPayload(ByteString.copyFrom(new byte[]{0})))) - .build() + .build() ) - .addVertex( - VertexPlan.newBuilder() - .setName("vertex4") - .setType(PlanVertexType.NORMAL) - .addInputs( - RootInputLeafOutputProto.newBuilder() - .setControllerDescriptor( - TezEntityDescriptorProto.newBuilder().setClassName( - initializerClassName)) - .setName("input4") - .setIODescriptor( - TezEntityDescriptorProto.newBuilder() - .setClassName("InputClazz") - .build() - ) - .build() - ) - .setTaskConfig( - PlanTaskConfiguration.newBuilder() - .setNumTasks(-1) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("x3.y3") + .addVertex( + VertexPlan.newBuilder() + .setName("vertex4") + .setType(PlanVertexType.NORMAL) + .addInputs( + RootInputLeafOutputProto.newBuilder() + .setControllerDescriptor( + TezEntityDescriptorProto.newBuilder().setClassName( + initializerClassName)) + .setName("input4") + .setIODescriptor( + TezEntityDescriptorProto.newBuilder() + .setClassName("InputClazz") .build() ) - .setVertexManagerPlugin(TezEntityDescriptorProto.newBuilder() - .setClassName(RootInputSpecUpdaterVertexManager.class.getName()) - .setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder() - .setUserPayload(ByteString.copyFrom(new byte[]{1})))) .build() ) + .setTaskConfig( + PlanTaskConfiguration.newBuilder() + .setNumTasks(-1) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("x3.y3") + .build() + ) + .setVertexManagerPlugin(TezEntityDescriptorProto.newBuilder() + .setClassName(RootInputSpecUpdaterVertexManager.class.getName()) + .setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder() + .setUserPayload(ByteString.copyFrom(new byte[]{1})))) + .build() + ) .addEdge( EdgePlan.newBuilder() .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("v1_v2")) @@ -1305,7 +1305,6 @@ private DAGPlan createDAGPlanWithRunningInitializer4() { return dag; } - private DAGPlan createDAGPlanWithRunningInitializer() { LOG.info("Setting up dag plan with running input initializer"); DAGPlan dag = DAGPlan.newBuilder() @@ -1370,7 +1369,7 @@ private DAGPlan createDAGPlanWithRunningInitializer() { .build(); return dag; } - + private DAGPlan createDAGPlanWithInputDistributor(String initializerClassName) { LOG.info("Setting up invalid dag plan with input distributor"); DAGPlan dag = DAGPlan.newBuilder() @@ -1387,11 +1386,11 @@ private DAGPlan createDAGPlanWithInputDistributor(String initializerClassName) { .setName("input1") .setIODescriptor( TezEntityDescriptorProto.newBuilder() - .setClassName("InputClazz") - .build() + .setClassName("InputClazz") + .build() ) .build() - ) + ) .setTaskConfig( PlanTaskConfiguration.newBuilder() .setNumTasks(2) @@ -1402,7 +1401,7 @@ private DAGPlan createDAGPlanWithInputDistributor(String initializerClassName) { .build() ) .addOutEdgeId("e1") - .build() + .build() ) .addVertex( VertexPlan.newBuilder() @@ -1410,15 +1409,15 @@ private DAGPlan createDAGPlanWithInputDistributor(String initializerClassName) { .setType(PlanVertexType.NORMAL) .setTaskConfig( PlanTaskConfiguration.newBuilder() - .setNumTasks(2) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("x1.y1") - .build() + .setNumTasks(2) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("x1.y1") + .build() ) .addInEdgeId("e1") - .build() + .build() ) .addEdge( EdgePlan.newBuilder() @@ -1431,18 +1430,18 @@ private DAGPlan createDAGPlanWithInputDistributor(String initializerClassName) { .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) .build() - ) + ) .build(); return dag; } - - private DAGPlan createDAGPlanForOneToOneSplit(String initializerClassName, - int numTasks, boolean addNullEdge) { + + private DAGPlan createDAGPlanForOneToOneSplit(String initializerClassName, + int numTasks, boolean addNullEdge) { VertexPlan.Builder v1Builder = VertexPlan.newBuilder(); v1Builder.setName("vertex1") - .setType(PlanVertexType.NORMAL) - .addOutEdgeId("e1") - .addOutEdgeId("e2"); + .setType(PlanVertexType.NORMAL) + .addOutEdgeId("e1") + .addOutEdgeId("e2"); if (addNullEdge) { v1Builder.addOutEdgeId("e5"); } @@ -1450,57 +1449,57 @@ private DAGPlan createDAGPlanForOneToOneSplit(String initializerClassName, numTasks = -1; v1Builder.addInputs( RootInputLeafOutputProto.newBuilder() - .setControllerDescriptor( - TezEntityDescriptorProto.newBuilder().setClassName( - initializerClassName)) - .setName("input1") - .setIODescriptor( - TezEntityDescriptorProto.newBuilder() - .setClassName("InputClazz") - .build() - ).build() + .setControllerDescriptor( + TezEntityDescriptorProto.newBuilder().setClassName( + initializerClassName)) + .setName("input1") + .setIODescriptor( + TezEntityDescriptorProto.newBuilder() + .setClassName("InputClazz") + .build() + ).build() ).setTaskConfig( PlanTaskConfiguration.newBuilder() - .setNumTasks(-1) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("x1.y1") - .build() + .setNumTasks(-1) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("x1.y1") + .build() ); } else { v1Builder.setTaskConfig( PlanTaskConfiguration.newBuilder() - .setNumTasks(numTasks) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("x1.y1") - .build() + .setNumTasks(numTasks) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("x1.y1") + .build() ); } VertexPlan v1Plan = v1Builder.build(); - + VertexPlan.Builder v4Builder = VertexPlan.newBuilder(); v4Builder - .setName("vertex4") - .setType(PlanVertexType.NORMAL) - .setTaskConfig( - PlanTaskConfiguration.newBuilder() - .setNumTasks(numTasks) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("x4.y4") - .build() - ) - .addInEdgeId("e3") - .addInEdgeId("e4"); + .setName("vertex4") + .setType(PlanVertexType.NORMAL) + .setTaskConfig( + PlanTaskConfiguration.newBuilder() + .setNumTasks(numTasks) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("x4.y4") + .build() + ) + .addInEdgeId("e3") + .addInEdgeId("e4"); if (addNullEdge) { v4Builder.addOutEdgeId("e6"); } VertexPlan v4Plan = v4Builder.build(); - + LOG.info("Setting up one to one dag plan"); DAGPlan.Builder dagBuilder = DAGPlan.newBuilder() .setName("testVertexOneToOneSplit") @@ -1511,16 +1510,16 @@ private DAGPlan createDAGPlanForOneToOneSplit(String initializerClassName, .setType(PlanVertexType.NORMAL) .setTaskConfig( PlanTaskConfiguration.newBuilder() - .setNumTasks(numTasks) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("x2.y2") - .build() + .setNumTasks(numTasks) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("x2.y2") + .build() ) .addInEdgeId("e1") .addOutEdgeId("e3") - .build() + .build() ) .addVertex( VertexPlan.newBuilder() @@ -1528,16 +1527,16 @@ private DAGPlan createDAGPlanForOneToOneSplit(String initializerClassName, .setType(PlanVertexType.NORMAL) .setTaskConfig( PlanTaskConfiguration.newBuilder() - .setNumTasks(numTasks) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("x3.y3") - .build() + .setNumTasks(numTasks) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("x3.y3") + .build() ) .addInEdgeId("e2") .addOutEdgeId("e4") - .build() + .build() ) .addVertex(v4Plan) .addEdge( @@ -1586,39 +1585,39 @@ private DAGPlan createDAGPlanForOneToOneSplit(String initializerClassName, .setId("e4") .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) - - ); + + ); if (addNullEdge) { dagBuilder.addVertex( VertexPlan.newBuilder() - .setName("vertex5") - .setType(PlanVertexType.NORMAL) - .setTaskConfig( - PlanTaskConfiguration.newBuilder() - .setNumTasks(1) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("x4.y4") + .setName("vertex5") + .setType(PlanVertexType.NORMAL) + .setTaskConfig( + PlanTaskConfiguration.newBuilder() + .setNumTasks(1) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("x4.y4") + .build() + ) + .addInEdgeId("e5") .build() - ) - .addInEdgeId("e5") - .build() ).addVertex( VertexPlan.newBuilder() - .setName("vertex6") - .setType(PlanVertexType.NORMAL) - .setTaskConfig( - PlanTaskConfiguration.newBuilder() - .setNumTasks(1) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("x4.y4") + .setName("vertex6") + .setType(PlanVertexType.NORMAL) + .setTaskConfig( + PlanTaskConfiguration.newBuilder() + .setNumTasks(1) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("x4.y4") + .build() + ) + .addInEdgeId("e6") .build() - ) - .addInEdgeId("e6") - .build() ) .addEdge( EdgePlan.newBuilder() @@ -1654,46 +1653,46 @@ private DAGPlan createTestDAGPlan() { DAGPlan dag = DAGPlan.newBuilder() .setName("testverteximpl") .addLocalResource( - PlanLocalResource.newBuilder() - .setName("dag lr") - .setUri("dag ir uri") - .setSize(1) - .setTimeStamp(1) - .setType(PlanLocalResourceType.FILE) - .setVisibility(PlanLocalResourceVisibility.APPLICATION) - .build() + PlanLocalResource.newBuilder() + .setName("dag lr") + .setUri("dag ir uri") + .setSize(1) + .setTimeStamp(1) + .setType(PlanLocalResourceType.FILE) + .setVisibility(PlanLocalResourceVisibility.APPLICATION) + .build() ) .addVertex( - VertexPlan.newBuilder() - .setName("vertex1") - .setType(PlanVertexType.NORMAL) - .addTaskLocationHint( - PlanTaskLocationHint.newBuilder() - .addHost("host1") - .addRack("rack1") - .build() - ) - .setTaskConfig( - PlanTaskConfiguration.newBuilder() - .setNumTasks(1) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("x1.y1") - .addLocalResource( - PlanLocalResource.newBuilder() - .setName("vertex lr") - .setUri("vertex ir uri") - .setSize(1) - .setTimeStamp(1) - .setType(PlanLocalResourceType.FILE) - .setVisibility(PlanLocalResourceVisibility.APPLICATION) - .build() + VertexPlan.newBuilder() + .setName("vertex1") + .setType(PlanVertexType.NORMAL) + .addTaskLocationHint( + PlanTaskLocationHint.newBuilder() + .addHost("host1") + .addRack("rack1") + .build() ) + .setTaskConfig( + PlanTaskConfiguration.newBuilder() + .setNumTasks(1) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("x1.y1") + .addLocalResource( + PlanLocalResource.newBuilder() + .setName("vertex lr") + .setUri("vertex ir uri") + .setSize(1) + .setTimeStamp(1) + .setType(PlanLocalResourceType.FILE) + .setVisibility(PlanLocalResourceVisibility.APPLICATION) + .build() + ) + .build() + ) + .addOutEdgeId("e1") .build() - ) - .addOutEdgeId("e1") - .build() ) .addVertex( VertexPlan.newBuilder() @@ -1744,50 +1743,50 @@ private DAGPlan createTestDAGPlan() { .build() ) .addVertex( - VertexPlan.newBuilder() - .setName("vertex4") - .setType(PlanVertexType.NORMAL) - .addTaskLocationHint( - PlanTaskLocationHint.newBuilder() - .addHost("host4") - .addRack("rack4") - .build() - ) - .setTaskConfig( - PlanTaskConfiguration.newBuilder() - .setNumTasks(2) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("x4.y4") - .build() - ) - .addInEdgeId("e3") - .addOutEdgeId("e5") - .build() + VertexPlan.newBuilder() + .setName("vertex4") + .setType(PlanVertexType.NORMAL) + .addTaskLocationHint( + PlanTaskLocationHint.newBuilder() + .addHost("host4") + .addRack("rack4") + .build() + ) + .setTaskConfig( + PlanTaskConfiguration.newBuilder() + .setNumTasks(2) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("x4.y4") + .build() + ) + .addInEdgeId("e3") + .addOutEdgeId("e5") + .build() ) .addVertex( - VertexPlan.newBuilder() - .setName("vertex5") - .setType(PlanVertexType.NORMAL) - .addTaskLocationHint( - PlanTaskLocationHint.newBuilder() - .addHost("host5") - .addRack("rack5") - .build() - ) - .setTaskConfig( - PlanTaskConfiguration.newBuilder() - .setNumTasks(2) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("x5.y5") - .build() - ) - .addInEdgeId("e4") - .addOutEdgeId("e6") - .build() + VertexPlan.newBuilder() + .setName("vertex5") + .setType(PlanVertexType.NORMAL) + .addTaskLocationHint( + PlanTaskLocationHint.newBuilder() + .addHost("host5") + .addRack("rack5") + .build() + ) + .setTaskConfig( + PlanTaskConfiguration.newBuilder() + .setNumTasks(2) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("x5.y5") + .build() + ) + .addInEdgeId("e4") + .addOutEdgeId("e6") + .build() ) .addVertex( VertexPlan.newBuilder() @@ -1822,85 +1821,85 @@ private DAGPlan createTestDAGPlan() { .addInEdgeId("e6") .build() ) - .addEdge( - EdgePlan.newBuilder() - .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i3_v1")) - .setInputVertexName("vertex1") - .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o1")) - .setOutputVertexName("vertex3") - .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) - .setId("e1") - .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) - .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) - .build() - ) - .addEdge( - EdgePlan.newBuilder() - .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i3_v2")) - .setInputVertexName("vertex2") - .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o2")) - .setOutputVertexName("vertex3") - .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) - .setId("e2") - .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) - .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) - .build() - ) - .addEdge( - EdgePlan.newBuilder() - .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i4_v3")) - .setInputVertexName("vertex3") - .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o3_v4")) - .setOutputVertexName("vertex4") - .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) - .setId("e3") - .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) - .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) - .build() - ) - .addEdge( - EdgePlan.newBuilder() - .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i5_v3")) - .setInputVertexName("vertex3") - .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o3_v5")) - .setOutputVertexName("vertex5") - .setDataMovementType(PlanEdgeDataMovementType.CUSTOM) - .setEdgeManager( - TezEntityDescriptorProto.newBuilder() - .setClassName(EdgeManagerForTest.class.getName()) - .setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder() - .setUserPayload((ByteString.copyFrom(edgePayload)))) - .build()) - .setId("e4") - .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) - .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) - .build() - ) - .addEdge( - EdgePlan.newBuilder() - .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i6_v4")) - .setInputVertexName("vertex4") - .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("org.apache.tez.o4")) - .setOutputVertexName("vertex6") - .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) - .setId("e5") - .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) - .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) - .build() - ) - .addEdge( - EdgePlan.newBuilder() - .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i6_v5")) - .setInputVertexName("vertex5") - .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("org.apache.tez.o5")) - .setOutputVertexName("vertex6") - .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) - .setId("e6") - .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) - .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) - .build() - ) - .build(); + .addEdge( + EdgePlan.newBuilder() + .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i3_v1")) + .setInputVertexName("vertex1") + .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o1")) + .setOutputVertexName("vertex3") + .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) + .setId("e1") + .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) + .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) + .build() + ) + .addEdge( + EdgePlan.newBuilder() + .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i3_v2")) + .setInputVertexName("vertex2") + .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o2")) + .setOutputVertexName("vertex3") + .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) + .setId("e2") + .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) + .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) + .build() + ) + .addEdge( + EdgePlan.newBuilder() + .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i4_v3")) + .setInputVertexName("vertex3") + .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o3_v4")) + .setOutputVertexName("vertex4") + .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) + .setId("e3") + .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) + .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) + .build() + ) + .addEdge( + EdgePlan.newBuilder() + .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i5_v3")) + .setInputVertexName("vertex3") + .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("o3_v5")) + .setOutputVertexName("vertex5") + .setDataMovementType(PlanEdgeDataMovementType.CUSTOM) + .setEdgeManager( + TezEntityDescriptorProto.newBuilder() + .setClassName(EdgeManagerForTest.class.getName()) + .setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder() + .setUserPayload((ByteString.copyFrom(edgePayload)))) + .build()) + .setId("e4") + .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) + .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) + .build() + ) + .addEdge( + EdgePlan.newBuilder() + .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i6_v4")) + .setInputVertexName("vertex4") + .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("org.apache.tez.o4")) + .setOutputVertexName("vertex6") + .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) + .setId("e5") + .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) + .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) + .build() + ) + .addEdge( + EdgePlan.newBuilder() + .setEdgeDestination(TezEntityDescriptorProto.newBuilder().setClassName("i6_v5")) + .setInputVertexName("vertex5") + .setEdgeSource(TezEntityDescriptorProto.newBuilder().setClassName("org.apache.tez.o5")) + .setOutputVertexName("vertex6") + .setDataMovementType(PlanEdgeDataMovementType.SCATTER_GATHER) + .setId("e6") + .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) + .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) + .build() + ) + .build(); return dag; } @@ -1911,86 +1910,85 @@ private DAGPlan createSamplerDAGPlan(boolean customEdge) { LOG.info("Setting up dag plan"); VertexPlan.Builder vCBuilder = VertexPlan.newBuilder(); vCBuilder.setName("C") - .setType(PlanVertexType.NORMAL) - .setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("C.class")) - .addTaskLocationHint( - PlanTaskLocationHint.newBuilder() - .addHost("host3") - .addRack("rack3") - .build() - ) - .setTaskConfig( - PlanTaskConfiguration.newBuilder() - .setNumTasks(customEdge ? -1 : 2) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("foo") - .setTaskModule("x3.y3") - .build() - ) - .setVertexManagerPlugin( - TezEntityDescriptorProto.newBuilder().setClassName( - VertexManagerPluginForTest.class.getName())) - .addInEdgeId("A_C") - .addInEdgeId("B_C"); + .setType(PlanVertexType.NORMAL) + .setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("C.class")) + .addTaskLocationHint( + PlanTaskLocationHint.newBuilder() + .addHost("host3") + .addRack("rack3") + .build() + ) + .setTaskConfig( + PlanTaskConfiguration.newBuilder() + .setNumTasks(customEdge ? -1 : 2) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("foo") + .setTaskModule("x3.y3") + .build() + ) + .setVertexManagerPlugin( + TezEntityDescriptorProto.newBuilder().setClassName( + VertexManagerPluginForTest.class.getName())) + .addInEdgeId("A_C") + .addInEdgeId("B_C"); if (customEdge) { vCBuilder.setVertexManagerPlugin(TezEntityDescriptorProto.newBuilder() .setClassName(VertexManagerPluginForTest.class.getName())); - } VertexPlan vCPlan = vCBuilder.build(); DAGPlan dag = DAGPlan.newBuilder() .setName("TestSamplerDAG") .addVertex( - VertexPlan.newBuilder() - .setName("A") - .setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("A.class")) - .setType(PlanVertexType.NORMAL) - .addTaskLocationHint( - PlanTaskLocationHint.newBuilder() - .addHost("host1") - .addRack("rack1") - .build() - ) - .setTaskConfig( - PlanTaskConfiguration.newBuilder() - .setNumTasks(1) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("A.class") + VertexPlan.newBuilder() + .setName("A") + .setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("A.class")) + .setType(PlanVertexType.NORMAL) + .addTaskLocationHint( + PlanTaskLocationHint.newBuilder() + .addHost("host1") + .addRack("rack1") + .build() + ) + .setTaskConfig( + PlanTaskConfiguration.newBuilder() + .setNumTasks(1) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("A.class") + .build() + ) + .addOutEdgeId("A_B") + .addOutEdgeId("A_C") .build() - ) - .addOutEdgeId("A_B") - .addOutEdgeId("A_C") - .build() ) .addVertex( - VertexPlan.newBuilder() - .setName("B") - .setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("B.class")) - .setType(PlanVertexType.NORMAL) - .addTaskLocationHint( - PlanTaskLocationHint.newBuilder() - .addHost("host2") - .addRack("rack2") - .build() - ) - .setTaskConfig( - PlanTaskConfiguration.newBuilder() - .setNumTasks(2) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("") + VertexPlan.newBuilder() + .setName("B") + .setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("B.class")) + .setType(PlanVertexType.NORMAL) + .addTaskLocationHint( + PlanTaskLocationHint.newBuilder() + .addHost("host2") + .addRack("rack2") + .build() + ) + .setTaskConfig( + PlanTaskConfiguration.newBuilder() + .setNumTasks(2) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("") + .build() + ) + .addInEdgeId("A_B") + .addOutEdgeId("B_C") .build() - ) - .addInEdgeId("A_B") - .addOutEdgeId("B_C") - .build() ) .addVertex( - vCPlan + vCPlan ) .addEdge( EdgePlan.newBuilder() @@ -2031,68 +2029,68 @@ private DAGPlan createSamplerDAGPlan(boolean customEdge) { .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) .build() - ) + ) .build(); return dag; } - + // Create a plan with 3 vertices: A, B, C. Group(A,B)->C private DAGPlan createVertexGroupDAGPlan() { LOG.info("Setting up group dag plan"); DAGPlan dag = DAGPlan.newBuilder() .setName("TestGroupDAG") .addVertex( - VertexPlan.newBuilder() - .setName("A") - .setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("A.class")) - .setType(PlanVertexType.NORMAL) - .setTaskConfig( - PlanTaskConfiguration.newBuilder() - .setNumTasks(1) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("A.class") + VertexPlan.newBuilder() + .setName("A") + .setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("A.class")) + .setType(PlanVertexType.NORMAL) + .setTaskConfig( + PlanTaskConfiguration.newBuilder() + .setNumTasks(1) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("A.class") + .build() + ) + .addOutEdgeId("A_C") .build() - ) - .addOutEdgeId("A_C") - .build() ) .addVertex( - VertexPlan.newBuilder() - .setName("B") - .setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("B.class")) - .setType(PlanVertexType.NORMAL) - .setTaskConfig( - PlanTaskConfiguration.newBuilder() - .setNumTasks(2) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("") + VertexPlan.newBuilder() + .setName("B") + .setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("B.class")) + .setType(PlanVertexType.NORMAL) + .setTaskConfig( + PlanTaskConfiguration.newBuilder() + .setNumTasks(2) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("") + .build() + ) + .addOutEdgeId("B_C") .build() - ) - .addOutEdgeId("B_C") - .build() ) .addVertex( - VertexPlan.newBuilder() - .setName("C") - .setType(PlanVertexType.NORMAL) - .setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("C.class")) - .setTaskConfig( - PlanTaskConfiguration.newBuilder() - .setNumTasks(2) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("foo") - .setTaskModule("x3.y3") + VertexPlan.newBuilder() + .setName("C") + .setType(PlanVertexType.NORMAL) + .setProcessorDescriptor(TezEntityDescriptorProto.newBuilder().setClassName("C.class")) + .setTaskConfig( + PlanTaskConfiguration.newBuilder() + .setNumTasks(2) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("foo") + .setTaskModule("x3.y3") + .build() + ) + .addInEdgeId("A_C") + .addInEdgeId("B_C") .build() - ) - .addInEdgeId("A_C") - .addInEdgeId("B_C") - .build() ) .addEdge( EdgePlan.newBuilder() @@ -2117,23 +2115,23 @@ private DAGPlan createVertexGroupDAGPlan() { .setDataSourceType(PlanEdgeDataSourceType.PERSISTED) .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) .build() - ) - .addVertexGroups( - PlanVertexGroupInfo.newBuilder(). + ) + .addVertexGroups( + PlanVertexGroupInfo.newBuilder(). setGroupName("Group"). addGroupMembers("A"). addGroupMembers("B"). addEdgeMergedInputs( PlanGroupInputEdgeInfo.newBuilder().setDestVertexName("C"). - setMergedInput( - TezEntityDescriptorProto.newBuilder(). - setClassName("Group.class") - .build()).build())) + setMergedInput( + TezEntityDescriptorProto.newBuilder(). + setClassName("Group.class") + .build()).build())) .build(); return dag; } - + private DAGPlan createDAGWithCustomVertexManager() { LOG.info("Setting up custom vertex manager dag plan"); DAGPlan dag = DAGPlan.newBuilder() @@ -2206,7 +2204,7 @@ private DAGPlan createDAGWithCustomVertexManager() { .setSchedulingType(PlanEdgeSchedulingType.SEQUENTIAL) .build() ).build(); - + return dag; } @@ -2453,7 +2451,7 @@ public DAGPlan createDAGPlanVertexShuffleDelete() { .setVertexConf(DAGProtos.ConfigurationProto.newBuilder() .addConfKeyValues(DAGProtos.PlanKeyValuePair.newBuilder() .setKey(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS) - .setValue(2+""))) + .setValue(2 + ""))) .addOutEdgeId("e1") .build() ) @@ -2663,7 +2661,7 @@ private void setupVertices(boolean cleanupShuffleDataAtVertexLevel) { for (int i = 0; i < vCnt; ++i) { VertexPlan vPlan = dagPlan.getVertex(i); String vName = vPlan.getName(); - TezVertexID vertexId = TezVertexID.getInstance(dagId, i+1); + TezVertexID vertexId = TezVertexID.getInstance(dagId, i + 1); VertexImpl v = null; VertexLocationHint locationHint = DagTypeConverters.convertFromDAGPlan( vPlan.getTaskLocationHintList()); @@ -2705,7 +2703,7 @@ private void parseVertexEdges() { Map outVertices = new HashMap(); - for(String inEdgeId : vertexPlan.getInEdgeIdList()){ + for (String inEdgeId : vertexPlan.getInEdgeIdList()) { EdgePlan edgePlan = edgePlans.get(inEdgeId); Vertex inVertex = this.vertices.get(edgePlan.getInputVertexName()); Edge edge = this.edges.get(inEdgeId); @@ -2714,7 +2712,7 @@ private void parseVertexEdges() { inVertices.put(inVertex, edge); } - for(String outEdgeId : vertexPlan.getOutEdgeIdList()){ + for (String outEdgeId : vertexPlan.getOutEdgeIdList()) { EdgePlan edgePlan = edgePlans.get(outEdgeId); Vertex outVertex = this.vertices.get(edgePlan.getOutputVertexName()); Edge edge = this.edges.get(outEdgeId); @@ -2736,7 +2734,6 @@ private void parseVertexEdges() { } } - public void setupPreDagCreation() { LOG.info("____________ RESETTING CURRENT DAG ____________"); conf = new Configuration(); @@ -2750,7 +2747,7 @@ public void setupPreDagCreation() { anyInt()); } - @SuppressWarnings({ "unchecked", "rawtypes" }) + @SuppressWarnings({"unchecked", "rawtypes"}) public void setupPostDagCreation(boolean cleanupShuffleDataAtVertexLevel) throws TezException { String dagName = "dag0"; taskCommunicatorManagerInterface = mock(TaskCommunicatorManagerInterface.class); @@ -2813,7 +2810,7 @@ public void setupPostDagCreation(boolean cleanupShuffleDataAtVertexLevel) throws Map localResources = new HashMap<>(); for (PlanLocalResource planLR : dagPlan.getLocalResourceList()) { localResources.put(planLR.getName(), - DagTypeConverters.convertPlanLocalResourceToLocalResource(planLR)); + DagTypeConverters.convertPlanLocalResourceToLocalResource(planLR)); } when(dag.getLocalResources()).thenReturn(localResources); @@ -2822,18 +2819,19 @@ public void setupPostDagCreation(boolean cleanupShuffleDataAtVertexLevel) throws doReturn(dag).when(appContext).getCurrentDAG(); execService = mock(ListeningExecutorService.class); final ListenableFuture mockFuture = mock(ListenableFuture.class); - + doAnswer(new Answer() { public ListenableFuture answer(InvocationOnMock invocation) { - Object[] args = invocation.getArguments(); - CallableEvent e = (CallableEvent) args[0]; - dispatcher.getEventHandler().handle(e); - return mockFuture; - }}) - .when(execService).submit((Callable) any()); - + Object[] args = invocation.getArguments(); + CallableEvent e = (CallableEvent) args[0]; + dispatcher.getEventHandler().handle(e); + return mockFuture; + } + }) + .when(execService).submit((Callable) any()); + MockClock clock = new MockClock(); - + doReturn(execService).when(appContext).getExecService(); doReturn(conf).when(appContext).getAMConf(); doReturn(new Credentials()).when(dag).getCredentials(); @@ -2845,7 +2843,7 @@ public ListenableFuture answer(InvocationOnMock invocation) { doReturn(historyEventHandler).when(appContext).getHistoryHandler(); doReturn(dispatcher.getEventHandler()).when(appContext).getEventHandler(); doReturn(clock).when(appContext).getClock(); - + vertexGroups = Maps.newHashMap(); for (PlanVertexGroupInfo groupInfo : dagPlan.getVertexGroupsList()) { vertexGroups.put(groupInfo.getGroupName(), new VertexGroupInfo(groupInfo)); @@ -2888,7 +2886,7 @@ public Vertex answer(InvocationOnMock invocation) throws Throwable { } parseVertexEdges(); - + for (Edge edge : edges.values()) { edge.initialize(); } @@ -2957,7 +2955,7 @@ private void initAllVertices(VertexState expectedState) { private void initVertex(VertexImpl v) { Assert.assertEquals(VertexState.NEW, v.getState()); dispatcher.getEventHandler().handle(new VertexEvent(v.getVertexId(), - VertexEventType.V_INIT)); + VertexEventType.V_INIT)); dispatcher.await(); } @@ -2974,10 +2972,10 @@ private void killVertex(VertexImpl v) { } private void startVertex(VertexImpl v, - boolean checkRunningState) { + boolean checkRunningState) { Assert.assertEquals(VertexState.INITED, v.getState()); dispatcher.getEventHandler().handle(new VertexEvent(v.getVertexId(), - VertexEventType.V_START)); + VertexEventType.V_START)); dispatcher.await(); if (checkRunningState) { Assert.assertEquals(VertexState.RUNNING, v.getState()); @@ -3044,7 +3042,7 @@ public void testVertexInit() throws AMUserCodeException { .getOutputDescriptor().getClassName())); } - @Test(timeout=5000) + @Test(timeout = 5000) public void testNonExistVertexManager() throws TezException { setupPreDagCreation(); dagPlan = createDAGPlanWithNonExistVertexManager(); @@ -3053,11 +3051,11 @@ public void testNonExistVertexManager() throws TezException { v1.handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_INIT)); Assert.assertEquals(VertexState.FAILED, v1.getState()); Assert.assertEquals(VertexTerminationCause.INIT_FAILURE, v1.getTerminationCause()); - Assert.assertTrue(StringUtils.join(v1.getDiagnostics(),"") + Assert.assertTrue(StringUtils.join(v1.getDiagnostics(), "") .contains("java.lang.ClassNotFoundException: non-exist-vertexmanager")); } - @Test(timeout=5000) + @Test(timeout = 5000) public void testNonExistInputInitializer() throws TezException { setupPreDagCreation(); dagPlan = createDAGPlanWithNonExistInputInitializer(); @@ -3066,11 +3064,11 @@ public void testNonExistInputInitializer() throws TezException { v1.handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_INIT)); Assert.assertEquals(VertexState.FAILED, v1.getState()); Assert.assertEquals(VertexTerminationCause.INIT_FAILURE, v1.getTerminationCause()); - Assert.assertTrue(StringUtils.join(v1.getDiagnostics(),"") + Assert.assertTrue(StringUtils.join(v1.getDiagnostics(), "") .contains("java.lang.ClassNotFoundException: non-exist-input-initializer")); } - @Test(timeout=5000) + @Test(timeout = 5000) public void testNonExistOutputCommitter() throws TezException { setupPreDagCreation(); dagPlan = createDAGPlanWithNonExistOutputCommitter(); @@ -3079,19 +3077,20 @@ public void testNonExistOutputCommitter() throws TezException { v1.handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_INIT)); Assert.assertEquals(VertexState.FAILED, v1.getState()); Assert.assertEquals(VertexTerminationCause.INIT_FAILURE, v1.getTerminationCause()); - Assert.assertTrue(StringUtils.join(v1.getDiagnostics(),"") + Assert.assertTrue(StringUtils.join(v1.getDiagnostics(), "") .contains("java.lang.ClassNotFoundException: non-exist-output-committer")); } class TestUpdateListener implements VertexStateUpdateListener { List events = Lists.newLinkedList(); + @Override public void onStateUpdated(VertexStateUpdate event) { events.add(event); } } - - @Test (timeout=5000) + + @Test(timeout = 5000) public void testVertexConfigureEvent() throws Exception { initAllVertices(VertexState.INITED); TestUpdateListener listener = new TestUpdateListener(); @@ -3106,8 +3105,8 @@ public void testVertexConfigureEvent() throws Exception { listener.events.get(0).getVertexState()); updateTracker.unregisterForVertexUpdates("vertex3", listener); } - - @Test (timeout=5000) + + @Test(timeout = 5000) public void testVertexConfigureEventWithReconfigure() throws Exception { useCustomInitializer = true; setupPreDagCreation(); @@ -3117,9 +3116,9 @@ public void testVertexConfigureEventWithReconfigure() throws Exception { TestUpdateListener listener = new TestUpdateListener(); updateTracker - .registerForVertexUpdates("vertex2", - EnumSet.of(org.apache.tez.dag.api.event.VertexState.CONFIGURED), - listener); + .registerForVertexUpdates("vertex2", + EnumSet.of(org.apache.tez.dag.api.event.VertexState.CONFIGURED), + listener); VertexImplWithControlledInitializerManager v1 = (VertexImplWithControlledInitializerManager) vertices .get("vertex1"); @@ -3143,7 +3142,7 @@ public void testVertexConfigureEventWithReconfigure() throws Exception { updateTracker.unregisterForVertexUpdates("vertex2", listener); } - @Test (timeout=5000) + @Test(timeout = 5000) public void testVertexConfigureEventBadReconfigure() { initAllVertices(VertexState.INITED); VertexImpl v3 = vertices.get("vertex3"); @@ -3159,8 +3158,8 @@ public void testVertexConfigureEventBadReconfigure() { Assert.assertTrue(e.getMessage().contains("invoked only after vertexReconfigurationPlanned")); } } - - @Test (timeout=5000) + + @Test(timeout = 5000) public void testVertexConfigureEventBadSetParallelism() throws Exception { initAllVertices(VertexState.INITED); VertexImpl v3 = vertices.get("vertex3"); @@ -3184,8 +3183,8 @@ public void testVertexStart() { VertexImpl v = vertices.get("vertex2"); startVertex(v); } - - @Test (timeout = 5000) + + @Test(timeout = 5000) public void testVertexGetTAAttempts() throws Exception { initAllVertices(VertexState.INITED); VertexImpl v1 = vertices.get("vertex1"); @@ -3194,7 +3193,7 @@ public void testVertexGetTAAttempts() throws Exception { startVertex(v2); VertexImpl v3 = vertices.get("vertex3"); VertexImpl v4 = vertices.get("vertex4"); - + Assert.assertEquals(VertexState.RUNNING, v4.getState()); Assert.assertEquals(1, v4.sourceVertices.size()); Edge e = v4.sourceVertices.get(v3); @@ -3202,10 +3201,10 @@ public void testVertexGetTAAttempts() throws Exception { TezTaskID.getInstance(v3.getVertexId(), 0), 0); TezTaskAttemptID v4TaId = TezTaskAttemptID.getInstance( TezTaskID.getInstance(v4.getVertexId(), 0), 0); - - for (int i=0; i<5; ++i) { + + for (int i = 0; i < 5; ++i) { v4.handle(new VertexEventRouteEvent(v4.getVertexId(), Collections.singletonList( - new TezEvent(DataMovementEvent.create(0, null), + new TezEvent(DataMovementEvent.create(0, null), new EventMetaData(EventProducerConsumerType.OUTPUT, v3.getName(), v3.getName(), v3TaId))))); } dispatcher.await(); @@ -3214,22 +3213,22 @@ public void testVertexGetTAAttempts() throws Exception { Assert.assertEquals(5, v4.pendingTaskEvents.size()); List taskList = new LinkedList(); // scheduling start to trigger edge routing to begin - for (int i=0; i 0 ? (lastFromEventId+=2) : (lastFromEventId+=3)), fromEventId); + Assert.assertEquals((i % 2 > 0 ? (lastFromEventId += 2) : (lastFromEventId += 3)), fromEventId); Assert.assertEquals(5, eventInfo.getEvents().size()); } eventInfo = v4.getTaskAttemptTezEvents(v4TaId, fromEventId, 0, 5); Assert.assertEquals(11, eventInfo.getNextFromEventId()); // all events traversed Assert.assertEquals(2, eventInfo.getEvents().size()); // remainder events } - - @Test (timeout = 5000) + + @Test(timeout = 5000) public void testVertexGetTAAttemptsObsoletion() throws Exception { initAllVertices(VertexState.INITED); VertexImpl v1 = vertices.get("vertex1"); @@ -3302,10 +3301,10 @@ public void testVertexGetTAAttemptsObsoletion() throws Exception { startVertex(v2); VertexImpl v3 = vertices.get("vertex3"); VertexImpl v4 = vertices.get("vertex4"); - + List taskList = new LinkedList(); // scheduling start to trigger edge routing to begin - for (int i=0; i taskList = new LinkedList(); // scheduling start to trigger edge routing to begin - for (int i=0; i edgeManagerDescriptors = Collections.singletonMap( - v1.getName(), edgeProp); + v1.getName(), edgeProp); v3.reconfigureVertex(1, null, edgeManagerDescriptors); v3.doneReconfiguringVertex(); assertTrue(v3.sourceVertices.get(v1).getEdgeManager() instanceof @@ -3499,19 +3498,19 @@ public void testVertexSetParallelismIncrease() throws Exception { EdgeManagerPluginDescriptor mockEdgeManagerDescriptor = EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName()); - EdgeProperty edgeProp = EdgeProperty.create(mockEdgeManagerDescriptor, - DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), + EdgeProperty edgeProp = EdgeProperty.create(mockEdgeManagerDescriptor, + DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")); Map edgeManagerDescriptors = Collections.singletonMap( - v1.getName(), edgeProp); + v1.getName(), edgeProp); v3.reconfigureVertex(10, null, edgeManagerDescriptors); v3.doneReconfiguringVertex(); assertTrue(v3.sourceVertices.get(v1).getEdgeManager() instanceof EdgeManagerForTest); checkTasks(v3, 10); } - + @Test(timeout = 5000) public void testVertexSetParallelismMultiple() throws Exception { VertexImpl v3 = vertices.get("vertex3"); @@ -3526,7 +3525,7 @@ public void testVertexSetParallelismMultiple() throws Exception { startVertex(v1); v3.reconfigureVertex(10, null, null); checkTasks(v3, 10); - + v3.reconfigureVertex(5, null, null); checkTasks(v3, 5); v3.doneReconfiguringVertex(); @@ -3578,7 +3577,7 @@ public void testVertexSetParallelismMultipleFailAfterSchedule() throws Exception Assert.assertTrue(e.getMessage().contains("setParallelism cannot be called after scheduling")); } } - + @Test(timeout = 5000) public void testVertexScheduleSendEvent() throws Exception { VertexImpl v3 = vertices.get("vertex3"); @@ -3603,7 +3602,7 @@ public void testVertexScheduleSendEvent() throws Exception { Assert.assertNotNull(event.getBaseTaskSpec()); Assert.assertEquals("foobar", event.getBaseTaskSpec().getTaskConf().get("abc")); } - + @Test(timeout = 5000) public void testVertexSetParallelismFailAfterSchedule() throws Exception { VertexImpl v3 = vertices.get("vertex3"); @@ -3624,7 +3623,7 @@ public void testVertexSetParallelismFailAfterSchedule() throws Exception { Assert.assertTrue(e.getMessage().contains("setParallelism cannot be called after scheduling")); } } - + @Test(timeout = 5000) public void testVertexPendingTaskEvents() { // Remove after bulk routing API is removed @@ -3632,9 +3631,9 @@ public void testVertexPendingTaskEvents() { VertexImpl v3 = vertices.get("vertex3"); VertexImpl v2 = vertices.get("vertex2"); VertexImpl v1 = vertices.get("vertex1"); - + startVertex(v1); - + TezTaskID t0_v2 = TezTaskID.getInstance(v2.getVertexId(), 0); TezTaskAttemptID ta0_t0_v2 = TezTaskAttemptID.getInstance(t0_v2, 0); @@ -3677,8 +3676,8 @@ public void testSetCustomEdgeManager() throws Exception { EdgeManagerPluginDescriptor edgeManagerDescriptor = EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName()); edgeManagerDescriptor.setUserPayload(userPayload); - EdgeProperty edgeProp = EdgeProperty.create(edgeManagerDescriptor, - DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), + EdgeProperty edgeProp = EdgeProperty.create(edgeManagerDescriptor, + DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")); Vertex v3 = vertices.get("vertex3"); @@ -3937,7 +3936,7 @@ public void testVertexTaskAttemptProcessorFailure() throws Exception { dispatcher.await(); TaskAttemptImpl ta = (TaskAttemptImpl) v.getTask(0).getAttempts().values().iterator().next(); ta.handle(new TaskAttemptEventSchedule(ta.getTaskAttemptID(), 2, 2)); - + NodeId nid = NodeId.newInstance("127.0.0.1", 0); ContainerId contId = ContainerId.newInstance(appAttemptId, 3); Container container = mock(Container.class); @@ -3972,7 +3971,7 @@ public void testVertexTaskAttemptInputFailure() throws Exception { dispatcher.await(); TaskAttemptImpl ta = (TaskAttemptImpl) v.getTask(0).getAttempts().values().iterator().next(); ta.handle(new TaskAttemptEventSchedule(ta.getTaskAttemptID(), 2, 2)); - + NodeId nid = NodeId.newInstance("127.0.0.1", 0); ContainerId contId = ContainerId.newInstance(appAttemptId, 3); Container container = mock(Container.class); @@ -3997,7 +3996,6 @@ public void testVertexTaskAttemptInputFailure() throws Exception { Assert.assertEquals(TaskAttemptTerminationCause.INPUT_READ_ERROR, ta.getTerminationCause()); } - @SuppressWarnings("deprecation") @Test(timeout = 5000) public void testVertexTaskAttemptOutputFailure() throws Exception { @@ -4009,7 +4007,7 @@ public void testVertexTaskAttemptOutputFailure() throws Exception { dispatcher.await(); TaskAttemptImpl ta = (TaskAttemptImpl) v.getTask(0).getAttempts().values().iterator().next(); ta.handle(new TaskAttemptEventSchedule(ta.getTaskAttemptID(), 2, 2)); - + NodeId nid = NodeId.newInstance("127.0.0.1", 0); ContainerId contId = ContainerId.newInstance(appAttemptId, 3); Container container = mock(Container.class); @@ -4033,7 +4031,7 @@ public void testVertexTaskAttemptOutputFailure() throws Exception { Assert.assertEquals(VertexState.RUNNING, v.getState()); Assert.assertEquals(TaskAttemptTerminationCause.OUTPUT_WRITE_ERROR, ta.getTerminationCause()); } - + @Test(timeout = 5000) public void testSourceVertexStartHandling() { LOG.info("Testing testSourceVertexStartHandling"); @@ -4108,7 +4106,6 @@ public void testSourceTaskAttemptCompletionEvents() { Assert.assertEquals(VertexState.SUCCEEDED, v5.getState()); Assert.assertEquals(VertexState.RUNNING, v6.getState()); Assert.assertEquals(4, v6.numSuccessSourceAttemptCompletions); - } @Test(timeout = 5000) @@ -4269,10 +4266,9 @@ public void testTaskReschedule() { new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED)); dispatcher.await(); Assert.assertEquals(VertexState.SUCCEEDED, v.getState()); - } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testTerminatingVertexForTaskComplete() throws Exception { setupPreDagCreation(); dagPlan = createSamplerDAGPlan(false); @@ -4290,7 +4286,7 @@ public void testTerminatingVertexForTaskComplete() throws Exception { } } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testTerminatingVertexForVComplete() throws Exception { setupPreDagCreation(); dagPlan = createSamplerDAGPlan(false); @@ -4360,7 +4356,7 @@ public void testVertexSuccessToFailedAfterTaskScheduler() throws Exception { outputs.add(RootInputLeafOutputProto.newBuilder() .setControllerDescriptor( TezEntityDescriptorProto.newBuilder().setClassName( - CountingOutputCommitter.class.getName()) + CountingOutputCommitter.class.getName()) .setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder() .setUserPayload(ByteString.copyFrom( new CountingOutputCommitter.CountingOutputCommitterConfig() @@ -4448,14 +4444,13 @@ public void testTaskFailedAfterVertexSuccess() { Assert.assertEquals(0, committer.abortCounter); Assert.assertEquals(1, committer.initCounter); Assert.assertEquals(1, committer.setupCounter); - + dispatcher.getEventHandler().handle( new VertexEventTaskCompleted(t2, TaskState.FAILED)); dispatcher.await(); Assert.assertEquals(VertexState.FAILED, v.getState()); Assert.assertEquals(1, committer.commitCounter); Assert.assertEquals(1, committer.abortCounter); - } @Test(timeout = 5000) @@ -4467,11 +4462,11 @@ public void testBadCommitter() throws Exception { outputs.add(RootInputLeafOutputProto.newBuilder() .setControllerDescriptor( TezEntityDescriptorProto.newBuilder().setClassName( - CountingOutputCommitter.class.getName()) + CountingOutputCommitter.class.getName()) .setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder() - .setUserPayload(ByteString.copyFrom( - new CountingOutputCommitter.CountingOutputCommitterConfig( - true, true, false).toUserPayload())).build())) + .setUserPayload(ByteString.copyFrom( + new CountingOutputCommitter.CountingOutputCommitterConfig( + true, true, false).toUserPayload())).build())) .setName("output_v2") .setIODescriptor( TezEntityDescriptorProto.newBuilder().setClassName("output.class")) @@ -4511,11 +4506,11 @@ public void testBadCommitter2() throws Exception { outputs.add(RootInputLeafOutputProto.newBuilder() .setControllerDescriptor( TezEntityDescriptorProto.newBuilder().setClassName( - CountingOutputCommitter.class.getName()) + CountingOutputCommitter.class.getName()) .setTezUserPayload(DAGProtos.TezUserPayloadProto.newBuilder() - .setUserPayload(ByteString.copyFrom( - new CountingOutputCommitter.CountingOutputCommitterConfig( - true, true, true).toUserPayload())).build())) + .setUserPayload(ByteString.copyFrom( + new CountingOutputCommitter.CountingOutputCommitterConfig( + true, true, true).toUserPayload())).build())) .setName("output_v2") .setIODescriptor( TezEntityDescriptorProto.newBuilder().setClassName("output.class")) @@ -4532,9 +4527,9 @@ public void testBadCommitter2() throws Exception { TezTaskID t2 = TezTaskID.getInstance(v.getVertexId(), 1); dispatcher.getEventHandler().handle( - new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED)); + new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED)); dispatcher.getEventHandler().handle( - new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED)); + new VertexEventTaskCompleted(t2, TaskState.SUCCEEDED)); dispatcher.await(); Assert.assertEquals(VertexState.FAILED, v.getState()); Assert.assertEquals(VertexTerminationCause.COMMIT_FAILURE, v.getTerminationCause()); @@ -4550,7 +4545,7 @@ public void testVertexInitWithCustomVertexManager() throws Exception { setupPreDagCreation(); dagPlan = createDAGWithCustomVertexManager(); setupPostDagCreation(false); - + int numTasks = 3; VertexImpl v1 = vertices.get("v1"); VertexImpl v2 = vertices.get("v2"); @@ -4605,21 +4600,20 @@ public void testVertexManagerHeuristic() throws TezException { dagPlan = createDAGPlanWithMixedEdges(); setupPostDagCreation(false); initAllVertices(VertexState.INITED); - Assert.assertEquals(ImmediateStartVertexManager.class, + Assert.assertEquals(ImmediateStartVertexManager.class, vertices.get("vertex1").getVertexManager().getPlugin().getClass()); - Assert.assertEquals(ShuffleVertexManager.class, + Assert.assertEquals(ShuffleVertexManager.class, vertices.get("vertex2").getVertexManager().getPlugin().getClass()); - Assert.assertEquals(InputReadyVertexManager.class, + Assert.assertEquals(InputReadyVertexManager.class, vertices.get("vertex3").getVertexManager().getPlugin().getClass()); - Assert.assertEquals(ImmediateStartVertexManager.class, + Assert.assertEquals(ImmediateStartVertexManager.class, vertices.get("vertex4").getVertexManager().getPlugin().getClass()); - Assert.assertEquals(ImmediateStartVertexManager.class, + Assert.assertEquals(ImmediateStartVertexManager.class, vertices.get("vertex5").getVertexManager().getPlugin().getClass()); - Assert.assertEquals(InputReadyVertexManager.class, + Assert.assertEquals(InputReadyVertexManager.class, vertices.get("vertex6").getVertexManager().getPlugin().getClass()); } - @Test(timeout = 5000) public void testVertexWithOneToOneSplit() throws Exception { // create a diamond shaped dag with 1-1 edges. @@ -4629,15 +4623,15 @@ public void testVertexWithOneToOneSplit() throws Exception { setupPreDagCreation(); dagPlan = createDAGPlanForOneToOneSplit("TestInputInitializer", -1, true); setupPostDagCreation(false); - + int numTasks = 5; VertexImplWithControlledInitializerManager v1 = (VertexImplWithControlledInitializerManager) vertices .get("vertex1"); VertexImpl v5 = vertices.get("vertex5"); initVertex(v1); - + Assert.assertEquals(VertexState.INITIALIZING, v1.getState()); - + // setting the edge manager should vertex1 should not INIT/START it since // input initialization is not complete. v5 should be inited EdgeManagerPluginDescriptor mockEdgeManagerDescriptor = @@ -4648,7 +4642,7 @@ public void testVertexWithOneToOneSplit() throws Exception { dispatcher.await(); Assert.assertEquals(VertexState.INITIALIZING, v1.getState()); Assert.assertEquals(VertexState.INITED, vertices.get("vertex5").getState()); - + RootInputInitializerManagerControlled initializerManager1 = v1.getRootInputInitializerManager(); List v1Hints = createTaskLocationHints(numTasks); initializerManager1.completeInputInitialization(0, numTasks, v1Hints); @@ -4657,7 +4651,7 @@ public void testVertexWithOneToOneSplit() throws Exception { Assert.assertEquals(numTasks, v1.getTotalTasks()); Assert.assertEquals(RootInputVertexManager.class.getName(), v1 .getVertexManager().getPlugin().getClass().getName()); - for (int i=0; i < v1Hints.size(); ++i) { + for (int i = 0; i < v1Hints.size(); ++i) { Assert.assertEquals(v1Hints.get(i), v1.getTaskLocationHints()[i]); } Assert.assertEquals(true, initializerManager1.hasShutDown); @@ -4670,7 +4664,7 @@ public void testVertexWithOneToOneSplit() throws Exception { // v4, v6 still initializing since edge is null Assert.assertEquals(VertexState.INITIALIZING, vertices.get("vertex4").getState()); Assert.assertEquals(VertexState.INITIALIZING, vertices.get("vertex6").getState()); - + Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex1").getState()); Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex2").getState()); Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex3").getState()); @@ -4678,7 +4672,7 @@ public void testVertexWithOneToOneSplit() throws Exception { // v4, v6 still initializing since edge is null Assert.assertEquals(VertexState.INITIALIZING, vertices.get("vertex4").getState()); Assert.assertEquals(VertexState.INITIALIZING, vertices.get("vertex6").getState()); - + mockEdgeManagerDescriptor = EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName()); e = vertices.get("vertex6").sourceVertices.get(vertices.get("vertex4")); @@ -4688,7 +4682,7 @@ public void testVertexWithOneToOneSplit() throws Exception { Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex4").getState()); Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex6").getState()); } - + @Test(timeout = 5000) public void testVertexWithOneToOneSplitWhileRunning() throws Exception { int numTasks = 5; @@ -4699,7 +4693,7 @@ public void testVertexWithOneToOneSplitWhileRunning() throws Exception { VertexImpl v1 = vertices.get("vertex1"); v1.vertexReconfigurationPlanned(); initAllVertices(VertexState.INITED); - + // fudge vertex manager so that tasks dont start running v1.vertexManager = new VertexManager( VertexManagerPluginDescriptor.create(VertexManagerPluginForTest.class.getName()), @@ -4727,7 +4721,7 @@ public void testVertexWithOneToOneSplitWhileRunning() throws Exception { Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex3").getState()); Assert.assertEquals(VertexState.RUNNING, vertices.get("vertex4").getState()); } - + @Test(timeout = 5000) public void testVertexWithOneToOneSplitWhileInited() throws Exception { int numTasks = 5; @@ -4738,13 +4732,13 @@ public void testVertexWithOneToOneSplitWhileInited() throws Exception { VertexImpl v1 = vertices.get("vertex1"); v1.vertexReconfigurationPlanned(); initAllVertices(VertexState.INITED); - + // fudge vertex manager so that tasks dont start running v1.vertexManager = new VertexManager( VertexManagerPluginDescriptor.create(VertexManagerPluginForTest.class.getName()), UserGroupInformation.getCurrentUser(), v1, appContext, mock(StateChangeNotifier.class)); v1.vertexManager.initialize(); - + Assert.assertEquals(numTasks, vertices.get("vertex2").getTotalTasks()); Assert.assertEquals(numTasks, vertices.get("vertex3").getTotalTasks()); Assert.assertEquals(numTasks, vertices.get("vertex4").getTotalTasks()); @@ -4779,7 +4773,7 @@ public void testVertexVMErrorReport() throws Exception { setupPostDagCreation(false); VertexImpl v1 = vertices.get("vertex1"); initAllVertices(VertexState.INITED); - + // fudge vertex manager so that tasks dont start running // it is not calling reconfigurtionPlanned() but will call reconfigureVertex(). // the vertex is already fully configured. this causes exception and verify that @@ -4840,7 +4834,7 @@ public void testVertexWithInitializerFailure() throws Exception { .contains(VertexTerminationCause.ROOT_INPUT_INIT_FAILURE.name())); Assert.assertTrue(StringUtils.join(v1.getDiagnostics(), ",") .contains("MockInitializerFailed")); - + VertexImplWithControlledInitializerManager v2 = (VertexImplWithControlledInitializerManager) vertices.get("vertex2"); Assert.assertEquals(VertexState.INITIALIZING, v2.getState()); RootInputInitializerManagerControlled initializerManager2 = v2.getRootInputInitializerManager(); @@ -5002,12 +4996,11 @@ public void testInputInitializerEventMultipleAttempts() throws Exception { Assert.assertEquals(1, initializerWrapper.getFirstSuccessfulAttemptMap().size()); Assert.assertEquals(2, initializerWrapper.getPendingEvents().get(v1.getName()).size()); - // Get all tasks of vertex1 to succeed. for (TezTaskID taskId : v1.getTasks().keySet()) { // Make attempt 1 of every task succeed TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 1); - v1.handle( new VertexEventTaskAttemptCompleted(taskAttemptId, TaskAttemptStateInternal.SUCCEEDED)); + v1.handle(new VertexEventTaskAttemptCompleted(taskAttemptId, TaskAttemptStateInternal.SUCCEEDED)); v1.handle(new VertexEventTaskCompleted(taskId, TaskState.SUCCEEDED)); dispatcher.await(); v1.stateChangeNotifier.taskSucceeded(v1.getName(), taskId, taskAttemptId.getId()); @@ -5017,14 +5010,13 @@ public void testInputInitializerEventMultipleAttempts() throws Exception { // v3 would have processed an INIT event and moved into INITIALIZING state. // Since source tasks were complete - the events should have been consumed. // Initializer would have run, and processed events. - while (v3.getState() != VertexState.RUNNING) { + while (v3.getState() != VertexState.RUNNING) { Thread.sleep(10); } Assert.assertEquals(VertexState.RUNNING, v3.getState()); Assert.assertEquals(1, initializer.initializerEvents.size()); Assert.assertEquals(expected, initializer.initializerEvents.get(0).getUserPayload()); - } @Test(timeout = 10000) @@ -5080,11 +5072,10 @@ public void testInputInitializerEventsMultipleSources() throws Exception { Assert.assertEquals(1, initializerWrapper.getFirstSuccessfulAttemptMap().size()); Assert.assertEquals(1, initializerWrapper.getPendingEvents().get(v1.getName()).size()); - // Get all tasks of vertex1 to succeed. for (TezTaskID taskId : v1.getTasks().keySet()) { TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); - v1.handle( new VertexEventTaskAttemptCompleted(taskAttemptId, TaskAttemptStateInternal.SUCCEEDED)); + v1.handle(new VertexEventTaskAttemptCompleted(taskAttemptId, TaskAttemptStateInternal.SUCCEEDED)); v1.handle(new VertexEventTaskCompleted(taskId, TaskState.SUCCEEDED)); dispatcher.await(); v1.stateChangeNotifier.taskSucceeded(v1.getName(), taskId, taskAttemptId.getId()); @@ -5093,7 +5084,6 @@ public void testInputInitializerEventsMultipleSources() throws Exception { Assert.assertEquals(1, initializer.initializerEvents.size()); - // Test written based on this Assert.assertEquals(2, v2.getTotalTasks()); // Generate events from v2 to v3's initializer. 1 from task 0, 2 from task 1 @@ -5123,7 +5113,7 @@ public void testInputInitializerEventsMultipleSources() throws Exception { // Get all tasks of vertex1 to succeed. for (TezTaskID taskId : v2.getTasks().keySet()) { TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); - v2.handle( new VertexEventTaskAttemptCompleted(taskAttemptId, TaskAttemptStateInternal.SUCCEEDED)); + v2.handle(new VertexEventTaskAttemptCompleted(taskAttemptId, TaskAttemptStateInternal.SUCCEEDED)); v2.handle(new VertexEventTaskCompleted(taskId, TaskState.SUCCEEDED)); dispatcher.await(); v2.stateChangeNotifier.taskSucceeded(v2.getName(), taskId, taskAttemptId.getId()); @@ -5133,7 +5123,7 @@ public void testInputInitializerEventsMultipleSources() throws Exception { // v3 would have processed an INIT event and moved into INITIALIZING state. // Since source tasks were complete - the events should have been consumed. // Initializer would have run, and processed events. - while (v3.getState() != VertexState.RUNNING) { + while (v3.getState() != VertexState.RUNNING) { Thread.sleep(10); } Assert.assertEquals(VertexState.RUNNING, v3.getState()); @@ -5148,7 +5138,6 @@ public void testInputInitializerEventsMultipleSources() throws Exception { expectedPayloads.remove(initializerEvent.getUserPayload()); } Assert.assertEquals(0, expectedPayloads.size()); - } @Test(timeout = 10000) @@ -5202,7 +5191,7 @@ public void testInputInitializerEventNoDirectConnection() throws Exception { // Get all tasks of vertex1 to succeed. for (TezTaskID taskId : v1.getTasks().keySet()) { TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); - v1.handle( new VertexEventTaskAttemptCompleted(taskAttemptId, TaskAttemptStateInternal.SUCCEEDED)); + v1.handle(new VertexEventTaskAttemptCompleted(taskAttemptId, TaskAttemptStateInternal.SUCCEEDED)); v1.handle(new VertexEventTaskCompleted(taskId, TaskState.SUCCEEDED)); dispatcher.await(); v1.stateChangeNotifier.taskSucceeded(v1.getName(), taskId, taskAttemptId.getId()); @@ -5212,7 +5201,7 @@ public void testInputInitializerEventNoDirectConnection() throws Exception { // v3 would have processed an INIT event and moved into INITIALIZING state. // Since source tasks were complete - the events should have been consumed. // Initializer would have run, and processed events. - while (v3.getState() != VertexState.RUNNING) { + while (v3.getState() != VertexState.RUNNING) { Thread.sleep(10); } @@ -5275,7 +5264,7 @@ public void testInputInitializerEventsAtNew() throws Exception { // Get Vertex1 to succeed before Vertex2 is INITED. Contrived case ? This is likely a tiny race. for (TezTaskID taskId : v1.getTasks().keySet()) { TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 0); - TaskImpl task = (TaskImpl)v1.getTask(taskId); + TaskImpl task = (TaskImpl) v1.getTask(taskId); task.handle(new TaskEvent(taskId, TaskEventType.T_ATTEMPT_LAUNCHED)); task.handle(new TaskEventTASucceeded(taskAttemptId)); v1.handle(new VertexEventTaskAttemptCompleted(taskAttemptId, TaskAttemptStateInternal.SUCCEEDED)); @@ -5294,11 +5283,10 @@ public void testInputInitializerEventsAtNew() throws Exception { startVertex(v2); dispatcher.await(); - // v3 would have processed an INIT event and moved into INITIALIZING state. // Since source tasks were complete - the events should have been consumed. // Initializer would have run, and processed events. - while (v3.getState() != VertexState.RUNNING) { + while (v3.getState() != VertexState.RUNNING) { Thread.sleep(10); } Assert.assertEquals(VertexState.RUNNING, v3.getState()); @@ -5345,7 +5333,6 @@ public void testInputInitializerEvents() throws Exception { Assert.assertFalse(initializer.eventReceived.get()); Assert.assertFalse(initializer.initComplete.get()); - // Signal the initializer by sending an event - via vertex1 InputInitializerEvent event = InputInitializerEvent.create("vertex2", "input1", null); // Create taskId and taskAttemptId for the single task that exists in vertex1 @@ -5384,7 +5371,7 @@ public void testInputInitializerEvents() throws Exception { } // Will eventually go into RUNNING state, via INITED - while (v2.getState() != VertexState.RUNNING) { + while (v2.getState() != VertexState.RUNNING) { Thread.sleep(10); } @@ -5446,24 +5433,24 @@ public void testTaskSchedulingWithCustomEdges() throws TezException { assertTrue(r3.getState().equals(VertexState.RUNNING)); //Let us start M7; M5 should start not start as it is dependent on M8 as well - dispatcher.getEventHandler().handle(new VertexEvent(m7.getVertexId(),VertexEventType.V_START)); + dispatcher.getEventHandler().handle(new VertexEvent(m7.getVertexId(), VertexEventType.V_START)); dispatcher.await(); //M5 should be in INITED state, as it depends on M8 assertTrue(m5.getState().equals(VertexState.INITED)); - for(Task task : m5.getTasks().values()) { + for (Task task : m5.getTasks().values()) { assertTrue(task.getState().equals(TaskState.NEW)); } //Let us start M8; M5 should start now - dispatcher.getEventHandler().handle(new VertexEvent(m8.getVertexId(),VertexEventType.V_START)); + dispatcher.getEventHandler().handle(new VertexEvent(m8.getVertexId(), VertexEventType.V_START)); dispatcher.await(); assertTrue(m9.getState().equals(VertexState.SUCCEEDED)); //M5 in running state. All source vertices have started and are configured assertTrue(m5.getState().equals(VertexState.RUNNING)); - for(Task task : m5.getTasks().values()) { + for (Task task : m5.getTasks().values()) { assertTrue(task.getState().equals(TaskState.SCHEDULED)); } } @@ -5482,135 +5469,135 @@ private DAGPlan createCustomDAGWithCustomEdges() { */ DAGPlan dag = DAGPlan.newBuilder().setName("TestSamplerDAG") .addVertex(VertexPlan.newBuilder() - .setName("M2") - .setProcessorDescriptor( - TezEntityDescriptorProto.newBuilder().setClassName("M2.class")) - .setType(PlanVertexType.NORMAL) - .addTaskLocationHint( - PlanTaskLocationHint.newBuilder().addHost("host1").addRack("rack1").build()) - .setTaskConfig(PlanTaskConfiguration.newBuilder() - .setNumTasks(1) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("M2.class") - .build() - ) - .addOutEdgeId("M2_R3") + .setName("M2") + .setProcessorDescriptor( + TezEntityDescriptorProto.newBuilder().setClassName("M2.class")) + .setType(PlanVertexType.NORMAL) + .addTaskLocationHint( + PlanTaskLocationHint.newBuilder().addHost("host1").addRack("rack1").build()) + .setTaskConfig(PlanTaskConfiguration.newBuilder() + .setNumTasks(1) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("M2.class") .build() + ) + .addOutEdgeId("M2_R3") + .build() ) .addVertex(VertexPlan.newBuilder() - .setName("M8") - .setProcessorDescriptor( - TezEntityDescriptorProto.newBuilder().setClassName("M8.class")) - .setType(PlanVertexType.NORMAL) - .addTaskLocationHint( - PlanTaskLocationHint.newBuilder().addHost("host1").addRack("rack1").build()) - .setTaskConfig(PlanTaskConfiguration.newBuilder() - .setNumTasks(1) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("M8.class") - .build() - ) - .addOutEdgeId("M8_M5") + .setName("M8") + .setProcessorDescriptor( + TezEntityDescriptorProto.newBuilder().setClassName("M8.class")) + .setType(PlanVertexType.NORMAL) + .addTaskLocationHint( + PlanTaskLocationHint.newBuilder().addHost("host1").addRack("rack1").build()) + .setTaskConfig(PlanTaskConfiguration.newBuilder() + .setNumTasks(1) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("M8.class") .build() + ) + .addOutEdgeId("M8_M5") + .build() ) .addVertex(VertexPlan.newBuilder() - .setName("M9") - .setProcessorDescriptor( - TezEntityDescriptorProto.newBuilder().setClassName("M9.class")) - .setType(PlanVertexType.NORMAL) - .addTaskLocationHint( - PlanTaskLocationHint.newBuilder().addHost("host1").addRack("rack1").build()) - .setTaskConfig(PlanTaskConfiguration.newBuilder() - .setNumTasks(0) //Zero task vertex - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("M9.class") - .build() - ) - .addOutEdgeId("M9_M5") + .setName("M9") + .setProcessorDescriptor( + TezEntityDescriptorProto.newBuilder().setClassName("M9.class")) + .setType(PlanVertexType.NORMAL) + .addTaskLocationHint( + PlanTaskLocationHint.newBuilder().addHost("host1").addRack("rack1").build()) + .setTaskConfig(PlanTaskConfiguration.newBuilder() + .setNumTasks(0) //Zero task vertex + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("M9.class") .build() + ) + .addOutEdgeId("M9_M5") + .build() ) - .addVertex(VertexPlan.newBuilder() - .setName("R3") - .setProcessorDescriptor( - TezEntityDescriptorProto.newBuilder().setClassName("M2.class")) - .setType(PlanVertexType.NORMAL) - .addTaskLocationHint( - PlanTaskLocationHint.newBuilder().addHost("host2").addRack("rack1").build()) - .setTaskConfig(PlanTaskConfiguration.newBuilder() - .setNumTasks(10) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("R3.class") - .build() - ) - .addInEdgeId("M2_R3") - .addOutEdgeId("R3_M5") - .build() - ) .addVertex(VertexPlan.newBuilder() - .setName("M5") - .setProcessorDescriptor( - TezEntityDescriptorProto.newBuilder().setClassName("M5.class")) - .setType(PlanVertexType.NORMAL) - .addTaskLocationHint( - PlanTaskLocationHint.newBuilder().addHost("host3").addRack("rack1").build()) - .setTaskConfig(PlanTaskConfiguration.newBuilder() - .setNumTasks(10) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("M5.class") - .build() - ) - .addInEdgeId("R3_M5") - .addInEdgeId("M7_M5") - .addInEdgeId("M8_M5") - .addInEdgeId("M9_M5") - .addOutEdgeId("M5_R6") + .setName("R3") + .setProcessorDescriptor( + TezEntityDescriptorProto.newBuilder().setClassName("M2.class")) + .setType(PlanVertexType.NORMAL) + .addTaskLocationHint( + PlanTaskLocationHint.newBuilder().addHost("host2").addRack("rack1").build()) + .setTaskConfig(PlanTaskConfiguration.newBuilder() + .setNumTasks(10) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("R3.class") .build() + ) + .addInEdgeId("M2_R3") + .addOutEdgeId("R3_M5") + .build() ) .addVertex(VertexPlan.newBuilder() - .setName("M7") - .setProcessorDescriptor( - TezEntityDescriptorProto.newBuilder().setClassName("M7.class")) - .setType(PlanVertexType.NORMAL) - .addTaskLocationHint( - PlanTaskLocationHint.newBuilder().addHost("host4").addRack("rack1").build()) - .setTaskConfig(PlanTaskConfiguration.newBuilder() - .setNumTasks(10) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("M7.class") - .build() - ) - .addOutEdgeId("M7_M5") + .setName("M5") + .setProcessorDescriptor( + TezEntityDescriptorProto.newBuilder().setClassName("M5.class")) + .setType(PlanVertexType.NORMAL) + .addTaskLocationHint( + PlanTaskLocationHint.newBuilder().addHost("host3").addRack("rack1").build()) + .setTaskConfig(PlanTaskConfiguration.newBuilder() + .setNumTasks(10) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("M5.class") .build() + ) + .addInEdgeId("R3_M5") + .addInEdgeId("M7_M5") + .addInEdgeId("M8_M5") + .addInEdgeId("M9_M5") + .addOutEdgeId("M5_R6") + .build() ) .addVertex(VertexPlan.newBuilder() - .setName("R6") - .setProcessorDescriptor( - TezEntityDescriptorProto.newBuilder().setClassName("R6.class")) - .setType(PlanVertexType.NORMAL) - .addTaskLocationHint( - PlanTaskLocationHint.newBuilder().addHost("host3").addRack("rack1").build()) - .setTaskConfig(PlanTaskConfiguration.newBuilder() - .setNumTasks(1) - .setVirtualCores(4) - .setMemoryMb(1024) - .setJavaOpts("") - .setTaskModule("R6.class") - .build() - ) - .addInEdgeId("M5_R6") + .setName("M7") + .setProcessorDescriptor( + TezEntityDescriptorProto.newBuilder().setClassName("M7.class")) + .setType(PlanVertexType.NORMAL) + .addTaskLocationHint( + PlanTaskLocationHint.newBuilder().addHost("host4").addRack("rack1").build()) + .setTaskConfig(PlanTaskConfiguration.newBuilder() + .setNumTasks(10) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("M7.class") + .build() + ) + .addOutEdgeId("M7_M5") + .build() + ) + .addVertex(VertexPlan.newBuilder() + .setName("R6") + .setProcessorDescriptor( + TezEntityDescriptorProto.newBuilder().setClassName("R6.class")) + .setType(PlanVertexType.NORMAL) + .addTaskLocationHint( + PlanTaskLocationHint.newBuilder().addHost("host3").addRack("rack1").build()) + .setTaskConfig(PlanTaskConfiguration.newBuilder() + .setNumTasks(1) + .setVirtualCores(4) + .setMemoryMb(1024) + .setJavaOpts("") + .setTaskModule("R6.class") .build() + ) + .addInEdgeId("M5_R6") + .build() ) .addEdge( EdgePlan.newBuilder() @@ -5776,7 +5763,7 @@ public void testVertexWithInitializerSuccess() throws Exception { Assert.assertEquals(5, v1.getTotalTasks()); Assert.assertEquals(RootInputVertexManager.class.getName(), v1 .getVertexManager().getPlugin().getClass().getName()); - for (int i=0; i < v1Hints.size(); ++i) { + for (int i = 0; i < v1Hints.size(); ++i) { Assert.assertEquals(v1Hints.get(i), v1.getTaskLocationHints()[i]); } Assert.assertEquals(true, initializerManager1.hasShutDown); @@ -5785,33 +5772,33 @@ public void testVertexWithInitializerSuccess() throws Exception { Assert.assertEquals(1, inputSpecs.size()); Assert.assertEquals(1, inputSpecs.get(0).getPhysicalEdgeCount()); } - + List taskList = new LinkedList(); // scheduling start to trigger edge routing to begin - for (int i=0; i events = Lists.newLinkedList(); TezTaskID t0_v1 = TezTaskID.getInstance(v1.getVertexId(), 0); TezTaskAttemptID ta0_t0_v1 = TezTaskAttemptID.getInstance(t0_v1, 0); events.add(new TezEvent( VertexManagerEvent.create("vertex2", ByteBuffer.wrap(new byte[0])), new EventMetaData( - EventProducerConsumerType.PROCESSOR, "vertex1", "vertex2", - ta0_t0_v1))); + EventProducerConsumerType.PROCESSOR, "vertex1", "vertex2", + ta0_t0_v1))); events.add(new TezEvent(InputDataInformationEvent.createWithSerializedPayload(0, ByteBuffer.wrap(new byte[0])), new EventMetaData(EventProducerConsumerType.INPUT, "vertex2", @@ -5819,7 +5806,7 @@ public void testVertexWithInitializerSuccess() throws Exception { dispatcher.getEventHandler().handle( new VertexEventRouteEvent(v2.getVertexId(), events)); dispatcher.await(); - + RootInputInitializerManagerControlled initializerManager2 = v2.getRootInputInitializerManager(); List v2Hints = createTaskLocationHints(10); initializerManager2.completeInputInitialization(0, 10, v2Hints); @@ -5828,23 +5815,23 @@ public void testVertexWithInitializerSuccess() throws Exception { Assert.assertEquals(10, v2.getTotalTasks()); Assert.assertEquals(RootInputVertexManager.class.getName(), v2 .getVertexManager().getPlugin().getClass().getName()); - for (int i=0; i < v2Hints.size(); ++i) { + for (int i = 0; i < v2Hints.size(); ++i) { Assert.assertEquals(v2Hints.get(i), v2.getTaskLocationHints()[i]); } Assert.assertEquals(true, initializerManager2.hasShutDown); - + // scheduling start to trigger edge routing to begin taskList = new LinkedList(); // scheduling start to trigger edge routing to begin - for (int i=0; i v2Hints = createTaskLocationHints(10); initializerManager2.completeInputInitialization(0, 10, v2Hints); @@ -5917,7 +5904,7 @@ public void testVertexWithInitializerSuccessLegacyRouting() throws Exception { Assert.assertEquals(10, v2.getTotalTasks()); Assert.assertEquals(RootInputVertexManager.class.getName(), v2 .getVertexManager().getPlugin().getClass().getName()); - for (int i=0; i < v2Hints.size(); ++i) { + for (int i = 0; i < v2Hints.size(); ++i) { Assert.assertEquals(v2Hints.get(i), v2.getTaskLocationHints()[i]); } Assert.assertEquals(true, initializerManager2.hasShutDown); @@ -5930,7 +5917,6 @@ public void testVertexWithInitializerSuccessLegacyRouting() throws Exception { } } - @Test(timeout = 5000) public void testVertexWithInputDistributor() throws Exception { useCustomInitializer = true; @@ -5965,7 +5951,7 @@ public void testVertexWithInputDistributor() throws Exception { Assert.assertEquals(VertexState.INITED, v1.getState()); Assert.assertEquals(VertexState.INITED, v2.getState()); } - + @Test(timeout = 5000) public void testVertexRootInputSpecUpdateAll() throws Exception { useCustomInitializer = true; @@ -5988,7 +5974,7 @@ public void testVertexRootInputSpecUpdateAll() throws Exception { Assert.assertEquals(RootInputSpecUpdaterVertexManager.class.getName(), v3.getVertexManager() .getPlugin().getClass().getName()); Assert.assertEquals(true, initializerManager1.hasShutDown); - + for (int i = 0; i < expectedNumTasks; i++) { List inputSpecs = v3.getInputSpecList(i); Assert.assertEquals(1, inputSpecs.size()); @@ -6018,7 +6004,7 @@ public void testVertexRootInputSpecUpdatePerTask() throws Exception { Assert.assertEquals(RootInputSpecUpdaterVertexManager.class.getName(), v4.getVertexManager() .getPlugin().getClass().getName()); Assert.assertEquals(true, initializerManager1.hasShutDown); - + for (int i = 0; i < expectedNumTasks; i++) { List inputSpecs = v4.getInputSpecList(i); Assert.assertEquals(1, inputSpecs.size()); @@ -6142,10 +6128,10 @@ protected RootInputInitializerManager createRootInputInitializerManager( @SuppressWarnings("rawtypes") private static class VertexImplWithControlledInitializerManager extends VertexImpl { - + private final DrainDispatcher dispatcher; private RootInputInitializerManagerControlled rootInputInitializerManager; - + public VertexImplWithControlledInitializerManager(TezVertexID vertexId, VertexPlan vertexPlan, String vertexName, Configuration conf, @@ -6178,13 +6164,12 @@ protected RootInputInitializerManager createRootInputInitializerManager( } return rootInputInitializerManager; } - + RootInputInitializerManagerControlled getRootInputInitializerManager() { return rootInputInitializerManager; } } - private static class RootInputInitializerManagerWithRunningInitializer extends RootInputInitializerManager { @@ -6198,13 +6183,12 @@ public RootInputInitializerManagerWithRunningInitializer(Vertex vertex, AppConte this.presetInitializer = presetInitializer; } - @Override protected InputInitializer createInitializer( RootInputLeafOutput input, InputInitializerContext context) { if (presetInitializer instanceof ContextSettableInputInitialzier) { - ((ContextSettableInputInitialzier)presetInitializer).setContext(context); + ((ContextSettableInputInitialzier) presetInitializer).setContext(context); } return presetInitializer; } @@ -6292,7 +6276,7 @@ public void completeInputInitialization(int initializerIndex) { } public void completeInputInitialization(int initializerIndex, int targetTasks, - List locationHints) { + List locationHints) { List events = Lists.newArrayListWithCapacity(targetTasks + 1); InputConfigureVertexTasksEvent configEvent = InputConfigureVertexTasksEvent.create( @@ -6309,7 +6293,7 @@ public void completeInputInitialization(int initializerIndex, int targetTasks, } } - @Test(timeout=5000) + @Test(timeout = 5000) public void testVertexGroupInput() throws TezException { setupPreDagCreation(); dagPlan = createVertexGroupDAGPlan(); @@ -6320,14 +6304,14 @@ public void testVertexGroupInput() throws TezException { VertexImpl vC = vertices.get("C"); dispatcher.getEventHandler().handle(new VertexEvent(vA.getVertexId(), - VertexEventType.V_INIT)); + VertexEventType.V_INIT)); dispatcher.getEventHandler().handle(new VertexEvent(vB.getVertexId(), VertexEventType.V_INIT)); dispatcher.await(); - + Assert.assertNull(vA.getGroupInputSpecList()); Assert.assertNull(vB.getGroupInputSpecList()); - + List groupInSpec = vC.getGroupInputSpecList(); Assert.assertEquals(1, groupInSpec.size()); Assert.assertEquals("Group", groupInSpec.get(0).getGroupName()); @@ -6349,15 +6333,15 @@ public void testStartWithUninitializedCustomEdge() throws Exception { VertexImpl vC = vertices.get("C"); dispatcher.getEventHandler().handle(new VertexEvent(vA.getVertexId(), - VertexEventType.V_INIT)); + VertexEventType.V_INIT)); dispatcher.getEventHandler().handle(new VertexEvent(vA.getVertexId(), - VertexEventType.V_START)); + VertexEventType.V_START)); dispatcher.await(); Assert.assertEquals(VertexState.INITIALIZING, vA.getState()); Assert.assertEquals(VertexState.INITIALIZING, vB.getState()); Assert.assertEquals(VertexState.INITIALIZING, vC.getState()); - + // setting the edge manager should vA to start EdgeManagerPluginDescriptor mockEdgeManagerDescriptor = EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName()); @@ -6368,9 +6352,9 @@ public void testStartWithUninitializedCustomEdge() throws Exception { Assert.assertEquals(VertexState.RUNNING, vA.getState()); Assert.assertEquals(VertexState.INITIALIZING, vB.getState()); Assert.assertEquals(VertexState.INITIALIZING, vC.getState()); - - EdgeProperty edgeProp = EdgeProperty.create(mockEdgeManagerDescriptor, - DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), + + EdgeProperty edgeProp = EdgeProperty.create(mockEdgeManagerDescriptor, + DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")); Map edges = Maps.newHashMap(); edges.put("B", edgeProp); @@ -6392,7 +6376,7 @@ public void testVertexConfiguredDoneByVMBeforeEdgeDefined() throws Exception { setupPreDagCreation(); dagPlan = createSamplerDAGPlan(true); setupPostDagCreation(false); - + VertexImpl vA = vertices.get("A"); VertexImpl vB = vertices.get("B"); VertexImpl vC = vertices.get("C"); @@ -6407,19 +6391,19 @@ public void testVertexConfiguredDoneByVMBeforeEdgeDefined() throws Exception { vB.vertexManager = new VertexManager( VertexManagerPluginDescriptor.create(VertexManagerPluginForTest.class.getName()), UserGroupInformation.getCurrentUser(), vB, appContext, mock(StateChangeNotifier.class)); - + vB.vertexReconfigurationPlanned(); - + dispatcher.getEventHandler().handle(new VertexEvent(vA.getVertexId(), - VertexEventType.V_INIT)); + VertexEventType.V_INIT)); dispatcher.getEventHandler().handle(new VertexEvent(vA.getVertexId(), - VertexEventType.V_START)); + VertexEventType.V_START)); dispatcher.await(); Assert.assertEquals(VertexState.INITIALIZING, vA.getState()); Assert.assertEquals(VertexState.INITIALIZING, vB.getState()); Assert.assertEquals(VertexState.INITIALIZING, vC.getState()); - + // setting the edge manager should vA to start EdgeManagerPluginDescriptor mockEdgeManagerDescriptor = EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName()); @@ -6430,15 +6414,15 @@ public void testVertexConfiguredDoneByVMBeforeEdgeDefined() throws Exception { Assert.assertEquals(VertexState.RUNNING, vA.getState()); Assert.assertEquals(VertexState.INITIALIZING, vB.getState()); Assert.assertEquals(VertexState.INITIALIZING, vC.getState()); - + // vB is not configured yet. Edge to C is not configured. So it should not send configured event // even thought VM says its doneConfiguring vertex vB.doneReconfiguringVertex(); Assert.assertEquals(0, listener.events.size()); - + // complete configuration and verify getting configured signal from vB - EdgeProperty edgeProp = EdgeProperty.create(mockEdgeManagerDescriptor, - DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), + EdgeProperty edgeProp = EdgeProperty.create(mockEdgeManagerDescriptor, + DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In")); Map edges = Maps.newHashMap(); edges.put("B", edgeProp); @@ -6450,7 +6434,7 @@ public void testVertexConfiguredDoneByVMBeforeEdgeDefined() throws Exception { Assert.assertEquals(org.apache.tez.dag.api.event.VertexState.CONFIGURED, listener.events.get(0).getVertexState()); updateTracker.unregisterForVertexUpdates(vB.getName(), listener); - + Assert.assertEquals(VertexState.RUNNING, vA.getState()); Assert.assertEquals(VertexState.RUNNING, vB.getState()); Assert.assertEquals(VertexState.RUNNING, vC.getState()); @@ -6472,9 +6456,9 @@ public void testInitStartRace() throws TezException { VertexImpl vC = vertices.get("C"); dispatcher.getEventHandler().handle(new VertexEvent(vA.getVertexId(), - VertexEventType.V_INIT)); + VertexEventType.V_INIT)); dispatcher.getEventHandler().handle(new VertexEvent(vA.getVertexId(), - VertexEventType.V_START)); + VertexEventType.V_START)); dispatcher.await(); Assert.assertEquals(VertexState.RUNNING, vA.getState()); @@ -6546,7 +6530,6 @@ public void testTez2684() throws IOException, TezException { dispatcher.getEventHandler().handle(new VertexEvent(vB.getVertexId(), VertexEventType.V_START)); dispatcher.await(); Assert.assertEquals(VertexState.RUNNING, vC.getState()); - } @Test(timeout = 5000) @@ -6691,7 +6674,7 @@ public void testExceptionFromVM_Initialize() throws TezException { Assert.assertTrue(diagnostics.contains(VMExceptionLocation.Initialize.name())); Assert.assertEquals(VertexTerminationCause.AM_USERCODE_FAILURE, v1.getTerminationCause()); } - + @Test(timeout = 5000) public void testExceptionFromVM_OnRootVertexInitialized() throws Exception { useCustomInitializer = true; @@ -6715,7 +6698,7 @@ public void testExceptionFromVM_OnRootVertexInitialized() throws Exception { Assert.assertTrue(diagnostics.contains(VMExceptionLocation.OnRootVertexInitialized.name())); Assert.assertEquals(VertexTerminationCause.AM_USERCODE_FAILURE, v1.getTerminationCause()); } - + @Test(timeout = 5000) public void testExceptionFromVM_OnVertexStarted() throws Exception { useCustomInitializer = true; @@ -6735,14 +6718,14 @@ public void testExceptionFromVM_OnVertexStarted() throws Exception { dispatcher.getEventHandler().handle(new VertexEvent(v1.getVertexId(), VertexEventType.V_START)); dispatcher.await(); - + Assert.assertEquals(VertexManagerWithException.class, v1.vertexManager.getPlugin().getClass()); Assert.assertEquals(VertexState.FAILED, v1.getState()); String diagnostics = StringUtils.join(v1.getDiagnostics(), ","); Assert.assertTrue(diagnostics.contains(VMExceptionLocation.OnVertexStarted.name())); Assert.assertEquals(VertexTerminationCause.AM_USERCODE_FAILURE, v1.getTerminationCause()); } - + @Test(timeout = 5000) public void testExceptionFromVM_OnSourceTaskCompleted() throws Exception { useCustomInitializer = true; @@ -6778,7 +6761,7 @@ public void testExceptionFromVM_OnSourceTaskCompleted() throws Exception { Assert.assertTrue(diagnostics.contains(VMExceptionLocation.OnSourceTaskCompleted.name())); Assert.assertEquals(VertexTerminationCause.AM_USERCODE_FAILURE, v2.getTerminationCause()); } - + @Test(timeout = 5000) public void testExceptionFromVM_OnVertexManagerEventReceived() throws Exception { useCustomInitializer = true; @@ -6806,7 +6789,7 @@ public void testExceptionFromVM_OnVertexManagerEventReceived() throws Exception String diagnostics = StringUtils.join(v1.getDiagnostics(), ","); Assert.assertTrue(diagnostics.contains(VMExceptionLocation.OnVertexManagerEventReceived.name())); } - + @Test(timeout = 5000) public void testExceptionFromVM_OnVertexManagerVertexStateUpdated() throws Exception { useCustomInitializer = true; @@ -6832,7 +6815,7 @@ public void testExceptionFromVM_OnVertexManagerVertexStateUpdated() throws Excep assertTrue(diagnostics.contains(VMExceptionLocation.OnVertexManagerVertexStateUpdated.name())); Assert.assertEquals(VertexTerminationCause.AM_USERCODE_FAILURE, v2.getTerminationCause()); } - + @Test(timeout = 5000) public void testExceptionFromII_Initialize() throws InterruptedException, TezException { useCustomInitializer = true; @@ -6865,7 +6848,7 @@ public void testExceptionFromII_InitFailedAfterInitialized() throws Exception { setupPostDagCreation(false); VertexImplWithControlledInitializerManager v1 = - (VertexImplWithControlledInitializerManager)vertices.get("vertex1"); + (VertexImplWithControlledInitializerManager) vertices.get("vertex1"); initVertex(v1); RootInputInitializerManagerControlled initializerManager1 = v1.getRootInputInitializerManager(); initializerManager1.completeInputInitialization(0); @@ -6889,7 +6872,7 @@ public void testExceptionFromII_InitFailedAfterRunning() throws Exception { setupPostDagCreation(false); VertexImplWithControlledInitializerManager v1 = - (VertexImplWithControlledInitializerManager)vertices.get("vertex1"); + (VertexImplWithControlledInitializerManager) vertices.get("vertex1"); initVertex(v1); RootInputInitializerManagerControlled initializerManager1 = v1.getRootInputInitializerManager(); initializerManager1.completeInputInitialization(0); @@ -6948,7 +6931,7 @@ public void testExceptionFromII_HandleInputInitializerEvent() throws Exception { dispatcher.getEventHandler() .handle(new VertexEventRouteEvent(v1.getVertexId(), Collections.singletonList(tezEvent))); dispatcher.await(); - + // it would cause v2 fail as its II throw exception in handleInputInitializerEvent String diagnostics = StringUtils.join(v2.getDiagnostics(), ","); assertTrue(diagnostics.contains(IIExceptionLocation.HandleInputInitializerEvent.name())); @@ -7017,7 +7000,7 @@ public void testExceptionFromII_InitSucceededAfterInitFailure() throws Interrupt Assert.assertEquals(VertexTerminationCause.ROOT_INPUT_INIT_FAILURE, v2.getTerminationCause()); } - @Test (timeout=5000) + @Test(timeout = 5000) public void testCompletedStatsCache() { initAllVertices(VertexState.INITED); VertexImpl v = vertices.get("vertex2"); @@ -7030,7 +7013,6 @@ public void testCompletedStatsCache() { dispatcher.getEventHandler().handle(new VertexEventTaskCompleted(t1, TaskState.SUCCEEDED)); dispatcher.await(); - VertexStatistics stats = v.getStatistics(); //Ensure that task 0 is available in completed stats cache @@ -7045,15 +7027,15 @@ public void testCompletedStatsCache() { Assert.assertTrue(v.completedTasksStatsCache.taskSet.cardinality() == 0); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testRouteEvent_RecoveredEvent() throws IOException { doReturn(historyEventHandler).when(appContext).getHistoryHandler(); doReturn(true).when(appContext).isRecoveryEnabled(); initAllVertices(VertexState.INITED); - VertexImpl v1 = (VertexImpl)vertices.get("vertex1"); - VertexImpl v2 = (VertexImpl)vertices.get("vertex2"); - VertexImpl v3 = (VertexImpl)vertices.get("vertex3"); + VertexImpl v1 = (VertexImpl) vertices.get("vertex1"); + VertexImpl v2 = (VertexImpl) vertices.get("vertex2"); + VertexImpl v3 = (VertexImpl) vertices.get("vertex3"); startVertex(v1); startVertex(v2); TezTaskID taskId = TezTaskID.getInstance(v1.getVertexId(), 0); @@ -7083,7 +7065,7 @@ private void verifyHistoryEvents(List events, HistoryEventType for (DAGHistoryEvent event : events) { LOG.info(event.getHistoryEvent().getEventType() + ""); if (event.getHistoryEvent().getEventType() == eventType) { - actualTimes ++; + actualTimes++; } } Assert.assertEquals(actualTimes, expectedTimes); @@ -7116,7 +7098,7 @@ public void onVertexManagerEventReceived(VertexManagerEvent vmEvent) { @Override public void onRootVertexInitialized(String inputName, InputDescriptor inputDescriptor, - List events) { + List events) { Map map = new HashMap(); if (getContext().getUserPayload().deepCopyAsArray()[0] == 0) { map.put("input3", InputSpecUpdate.createAllTaskInputSpecUpdate(4)); @@ -7200,7 +7182,7 @@ public void onRootVertexInitialized(String inputName, InputDescriptor inputDescr @InterfaceAudience.Private public static class VertexManagerWithException extends - ImmediateStartVertexManager{ + ImmediateStartVertexManager { public static enum VMExceptionLocation { NoExceptionDoReconfigure, @@ -7211,13 +7193,13 @@ public static enum VMExceptionLocation { OnVertexManagerVertexStateUpdated, Initialize, } - + private VMExceptionLocation exLocation; - + public VertexManagerWithException(VertexManagerPluginContext context) { super(context); } - + @Override public void initialize() { super.initialize(); @@ -7230,16 +7212,16 @@ public void initialize() { getContext().vertexReconfigurationPlanned(); } } - + @Override public void onRootVertexInitialized(String inputName, - InputDescriptor inputDescriptor, List events) { + InputDescriptor inputDescriptor, List events) { if (this.exLocation == VMExceptionLocation.OnRootVertexInitialized) { throw new RuntimeException(this.exLocation.name()); } super.onRootVertexInitialized(inputName, inputDescriptor, events); } - + @Override public void onSourceTaskCompleted(TaskAttemptIdentifier attempt) { if (this.exLocation == VMExceptionLocation.OnSourceTaskCompleted) { @@ -7247,7 +7229,7 @@ public void onSourceTaskCompleted(TaskAttemptIdentifier attempt) { } super.onSourceTaskCompleted(attempt); } - + @Override public void onVertexStarted(List completions) { if (this.exLocation == VMExceptionLocation.OnVertexStarted) { @@ -7258,7 +7240,7 @@ public void onVertexStarted(List completions) { getContext().doneReconfiguringVertex(); } } - + @Override public void onVertexManagerEventReceived(VertexManagerEvent vmEvent) { super.onVertexManagerEventReceived(vmEvent); @@ -7266,7 +7248,7 @@ public void onVertexManagerEventReceived(VertexManagerEvent vmEvent) { throw new RuntimeException(this.exLocation.name()); } } - + @Override public void onVertexStateUpdated(VertexStateUpdate stateUpdate) { super.onVertexStateUpdated(stateUpdate); @@ -7276,7 +7258,7 @@ public void onVertexStateUpdated(VertexStateUpdate stateUpdate) { } } } - + public static enum IIExceptionLocation { Initialize, HandleInputInitializerEvent, @@ -7427,7 +7409,6 @@ public void testCounterLimits() { System.out.println(v.getDiagnostics()); Assert.assertTrue("Diagnostics should contain counter limits error message", StringUtils.join(v.getDiagnostics(), ",").contains("Counters limit exceeded")); - } @Test(timeout = 5000) @@ -7628,7 +7609,7 @@ private void runVertices() { private void startAttempts(Map attempts) { for (Map.Entry entry : attempts.entrySet()) { TezTaskAttemptID id = entry.getKey(); - TaskAttemptImpl taskAttempt = (TaskAttemptImpl)entry.getValue(); + TaskAttemptImpl taskAttempt = (TaskAttemptImpl) entry.getValue(); taskAttempt.handle(new TaskAttemptEventSchedule(id, 10, 10)); dispatcher.await(); ContainerId mockContainer = mock(ContainerId.class, RETURNS_DEEP_STUBS); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java index b1976c34a3..97bb386fa4 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl2.java @@ -87,7 +87,7 @@ public void testTaskLoggingOptsPerLogger() { expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "DEBUG" + "," + TezConstants.TEZ_CONTAINER_LOGGER_NAME); - for (int i = 0 ; i < testInfo.numTasks ; i++) { + for (int i = 0; i < testInfo.numTasks; i++) { ContainerContext containerContext = vertexWrapper .vertex.getContainerContext(i); String javaOpts = containerContext.getJavaOpts(); @@ -121,7 +121,7 @@ public void testTaskLoggingOptsSimple() { expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "DEBUG" + "," + TezConstants.TEZ_CONTAINER_LOGGER_NAME); - for (int i = 0 ; i < testInfo.numTasks ; i++) { + for (int i = 0; i < testInfo.numTasks; i++) { ContainerContext containerContext = vertexWrapper.vertex.getContainerContext(i); String javaOpts = containerContext.getJavaOpts(); assertTrue(javaOpts.contains(testInfo.initialJavaOpts)); @@ -161,7 +161,7 @@ public void testTaskSpecificLoggingOpts() { expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "INFO" + "," + TezConstants.TEZ_CONTAINER_LOGGER_NAME); - for (int i = 3 ; i < testInfo.numTasks ; i++) { + for (int i = 3; i < testInfo.numTasks; i++) { ContainerContext containerContext = vertexWrapper.vertex.getContainerContext(i); String javaOpts = containerContext.getJavaOpts(); @@ -186,7 +186,7 @@ public void testTaskSpecificLoggingOpts() { expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "DEBUG" + "," + TezConstants.TEZ_CONTAINER_LOGGER_NAME); - for (int i = 0 ; i < 3 ; i++) { + for (int i = 0; i < 3; i++) { ContainerContext containerContext = vertexWrapper.vertex.getContainerContext(i); String javaOpts = containerContext.getJavaOpts(); @@ -227,7 +227,7 @@ public void testTaskSpecificLoggingOpts2() { expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "WARN" + "," + TezConstants.TEZ_CONTAINER_LOGGER_NAME); - for (int i = 3 ; i < testInfo.numTasks ; i++) { + for (int i = 3; i < testInfo.numTasks; i++) { ContainerContext containerContext = vertexWrapper.vertex.getContainerContext(i); String javaOpts = containerContext.getJavaOpts(); @@ -252,7 +252,7 @@ public void testTaskSpecificLoggingOpts2() { expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "DEBUG" + "," + TezConstants.TEZ_CONTAINER_LOGGER_NAME); - for (int i = 0 ; i < 3 ; i++) { + for (int i = 0; i < 3; i++) { ContainerContext containerContext = vertexWrapper.vertex.getContainerContext(i); String javaOpts = containerContext.getJavaOpts(); @@ -342,7 +342,6 @@ public void testVertexExecutionContextOverride() { assertEquals(1, vertexWrapper.vertex.taskCommunicatorIdentifier); } - private static class ExecutionContextTestInfoHolder { static final String TASK_SCHEDULER_NAME_BASE = "TASK_SCHEDULER"; @@ -540,7 +539,6 @@ private static class VertexWrapper { Assert.assertEquals("bar2", vertex.vertexOnlyConf.get("foo1")); Assert.assertEquals("bar", vertex.vertexOnlyConf.get("foo")); } - } VertexWrapper(VertexPlan vertexPlan, Configuration conf) { diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexManager.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexManager.java index 058dc76164..f6f52953a7 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexManager.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -79,14 +79,14 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; -@SuppressWarnings({ "rawtypes", "unchecked" }) +@SuppressWarnings({"rawtypes", "unchecked"}) public class TestVertexManager { AppContext mockAppContext; ListeningExecutorService execService; Vertex mockVertex; EventHandler mockHandler; ArgumentCaptor requestCaptor; - + @Before public void setup() { mockAppContext = mock(AppContext.class, RETURNS_DEEP_STUBS); @@ -94,12 +94,13 @@ public void setup() { final ListenableFuture mockFuture = mock(ListenableFuture.class); doAnswer(new Answer() { public ListenableFuture answer(InvocationOnMock invocation) { - Object[] args = invocation.getArguments(); - CallableEvent e = (CallableEvent) args[0]; - new CallableEventDispatcher().handle(e); - return mockFuture; - }}) - .when(execService).submit((Callable) any()); + Object[] args = invocation.getArguments(); + CallableEvent e = (CallableEvent) args[0]; + new CallableEventDispatcher().handle(e); + return mockFuture; + } + }) + .when(execService).submit((Callable) any()); doReturn(execService).when(mockAppContext).getExecService(); mockVertex = mock(Vertex.class, RETURNS_DEEP_STUBS); doReturn("vertex1").when(mockVertex).getName(); @@ -109,7 +110,6 @@ public ListenableFuture answer(InvocationOnMock invocation) { mockAppContext.getCurrentDAG().getVertex(any(String.class)) .getTotalTasks()).thenReturn(1); requestCaptor = ArgumentCaptor.forClass(VertexEventInputDataInformation.class); - } public static class CheckUserPayloadVertexManagerPlugin extends VertexManagerPlugin { @@ -131,7 +131,7 @@ public void onRootVertexInitialized(String inputName, InputDescriptor inputDescr @Test(timeout = 5000) public void testVertexManagerPluginCtorAccessUserPayload() throws IOException, TezException { - byte[] randomUserPayload = {1,2,3}; + byte[] randomUserPayload = {1, 2, 3}; UserPayload userPayload = UserPayload.create(ByteBuffer.wrap(randomUserPayload)); VertexManager vm = new VertexManager( @@ -140,7 +140,6 @@ public void testVertexManagerPluginCtorAccessUserPayload() throws IOException, T mockVertex, mockAppContext, mock(StateChangeNotifier.class)); } - @Test(timeout = 5000) public void testOnRootVertexInitialized() throws Exception { Configuration conf = new Configuration(); @@ -183,7 +182,7 @@ public void testOnRootVertexInitialized2() throws Exception { new VertexManager( VertexManagerPluginDescriptor.create(CustomVertexManager.class .getName()), UserGroupInformation.getCurrentUser(), - mockVertex, mockAppContext, mock(StateChangeNotifier.class)); + mockVertex, mockAppContext, mock(StateChangeNotifier.class)); vm.initialize(); InputDescriptor id1 = mock(InputDescriptor.class); List events1 = new LinkedList(); @@ -196,7 +195,7 @@ public void testOnRootVertexInitialized2() throws Exception { verify(mockHandler, times(1)).handle(requestCaptor.capture()); List tezEventsAfterInput1 = requestCaptor.getValue().getEvents(); assertEquals(0, tezEventsAfterInput1.size()); - + InputDescriptor id2 = mock(InputDescriptor.class); List events2 = new LinkedList(); InputDataInformationEvent diEvent2 = @@ -213,21 +212,21 @@ public void testOnRootVertexInitialized2() throws Exception { for (TezEvent tezEvent : tezEventsAfterInput2) { edgeVertexSet.add(tezEvent.getDestinationInfo().getEdgeVertexName()); } - assertEquals(Sets.newHashSet("input1","input2"), edgeVertexSet); + assertEquals(Sets.newHashSet("input1", "input2"), edgeVertexSet); } @Test(timeout = 5000) public void testVMPluginCtxGetInputVertexGroup() throws Exception { VertexManager vm = - new VertexManager(VertexManagerPluginDescriptor.create(CustomVertexManager.class.getName()), - UserGroupInformation.getCurrentUser(), mockVertex, mockAppContext, - mock(StateChangeNotifier.class)); + new VertexManager(VertexManagerPluginDescriptor.create(CustomVertexManager.class.getName()), + UserGroupInformation.getCurrentUser(), mockVertex, mockAppContext, + mock(StateChangeNotifier.class)); assertTrue(vm.pluginContext.getInputVertexGroups().isEmpty()); String group = "group", v1 = "v1", v2 = "v2"; when(mockVertex.getGroupInputSpecList()) - .thenReturn(Arrays.asList(new GroupInputSpec(group, Arrays.asList(v1, v2), null))); + .thenReturn(Arrays.asList(new GroupInputSpec(group, Arrays.asList(v1, v2), null))); Map> groups = vm.pluginContext.getInputVertexGroups(); assertEquals(1, groups.size()); assertTrue(groups.containsKey(group)); @@ -239,11 +238,11 @@ public void testVMPluginCtxGetInputVertexGroup() throws Exception { @Test(timeout = 5000) public void testSendCustomProcessorEvent() throws Exception { VertexManager vm = - new VertexManager(VertexManagerPluginDescriptor.create(CustomVertexManager.class.getName()), - UserGroupInformation.getCurrentUser(), mockVertex, mockAppContext, - mock(StateChangeNotifier.class)); + new VertexManager(VertexManagerPluginDescriptor.create(CustomVertexManager.class.getName()), + UserGroupInformation.getCurrentUser(), mockVertex, mockAppContext, + mock(StateChangeNotifier.class)); ArgumentCaptor requestCaptor = - ArgumentCaptor.forClass(VertexEventRouteEvent.class); + ArgumentCaptor.forClass(VertexEventRouteEvent.class); when(mockVertex.getTotalTasks()).thenReturn(2); @@ -272,12 +271,12 @@ public void testSendCustomProcessorEvent() throws Exception { verify(mockHandler, never()).handle(requestCaptor.capture()); //events.add(); - byte[] payload = new byte[] {1,2,3}; + byte[] payload = new byte[]{1, 2, 3}; events.add(CustomProcessorEvent.create(ByteBuffer.wrap(payload))); vm.pluginContext.sendEventToProcessor(events, 1); verify(mockHandler, times(1)).handle(requestCaptor.capture()); CustomProcessorEvent cpe = - (CustomProcessorEvent)(requestCaptor.getValue().getEvents().get(0).getEvent()); + (CustomProcessorEvent) (requestCaptor.getValue().getEvents().get(0).getEvent()); // should be able to get payload any times for (int i = 0; i < 2; i++) { @@ -291,7 +290,7 @@ public void testSendCustomProcessorEvent() throws Exception { public static class CustomVertexManager extends VertexManagerPlugin { - private Map> cachedEventMap = new HashMap>(); + private Map> cachedEventMap = new HashMap>(); public CustomVertexManager(VertexManagerPluginContext context) { super(context); @@ -318,13 +317,13 @@ public void onVertexManagerEventReceived(VertexManagerEvent vmEvent) { */ @Override public void onRootVertexInitialized(String inputName, - InputDescriptor inputDescriptor, List events) { + InputDescriptor inputDescriptor, List events) { cachedEventMap.put(inputName, events); if (inputName.equals("input2")) { for (Map.Entry> entry : cachedEventMap.entrySet()) { List riEvents = Lists.newLinkedList(); for (Event event : events) { - riEvents.add((InputDataInformationEvent)event); + riEvents.add((InputDataInformationEvent) event); } getContext().addRootInputEvents(entry.getKey(), riEvents); } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexStats.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexStats.java index eef877a8c2..b116f0761d 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexStats.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexStats.java @@ -155,5 +155,4 @@ public void testBasicStats() { Assert.assertEquals(2, stats.shortestDurationTasks.size()); Assert.assertEquals(2, stats.longestDurationTasks.size()); } - } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/speculation/legacy/TestDataStatistics.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/speculation/legacy/TestDataStatistics.java index f4791a99e4..61581fa51a 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/speculation/legacy/TestDataStatistics.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/speculation/legacy/TestDataStatistics.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.dag.speculation.legacy; @@ -34,7 +34,7 @@ public void testEmptyDataStatistics() throws Exception { Assert.assertEquals(0, statistics.std(), TOL); Assert.assertEquals(Long.MAX_VALUE, statistics.outlier(1.0f), TOL); } - + @Test(timeout = 5000) public void testSingleEntryDataStatistics() throws Exception { DataStatistics statistics = new DataStatistics(17.29); @@ -44,7 +44,7 @@ public void testSingleEntryDataStatistics() throws Exception { Assert.assertEquals(0, statistics.std(), TOL); Assert.assertEquals(17.29, statistics.outlier(1.0f), TOL); } - + @Test(timeout = 5000) public void testMutiEntryDataStatistics() throws Exception { DataStatistics statistics = new DataStatistics(); @@ -55,8 +55,8 @@ public void testMutiEntryDataStatistics() throws Exception { Assert.assertEquals(36.0, statistics.var(), TOL); Assert.assertEquals(6.0, statistics.std(), TOL); Assert.assertEquals(29.0, statistics.outlier(1.0f), TOL); - } - + } + @Test(timeout = 5000) public void testUpdateStatistics() throws Exception { DataStatistics statistics = new DataStatistics(17); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestContainerLauncherManager.java b/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestContainerLauncherManager.java index 0bd41f4a60..43880e0a81 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestContainerLauncherManager.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestContainerLauncherManager.java @@ -14,7 +14,6 @@ package org.apache.tez.dag.app.launcher; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -227,7 +226,6 @@ public void testEventRouting() throws Exception { clr.handle(launchRequestEvent1); - ArgumentCaptor captor = ArgumentCaptor.forClass(ContainerLaunchRequest.class); verify(clr.getTestContainerLauncher(0)).launchContainer(captor.capture()); @@ -241,7 +239,6 @@ public void testEventRouting() throws Exception { assertEquals(1, captor.getAllValues().size()); ContainerLaunchRequest launchRequest2 = captor.getValue(); assertEquals(clc2, launchRequest2.getContainerLaunchContext()); - } finally { clr.stop(); verify(clr.getTestContainerLauncher(0)).shutdown(); @@ -278,7 +275,6 @@ public void testReportFailureFromContainerLauncher() throws ServicePluginExcepti ContainerLauncherLaunchRequestEvent launchRequestEvent = new ContainerLauncherLaunchRequestEvent(clc1, container1, 0, 0, 0); - containerLauncherManager.handle(launchRequestEvent); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Event.class); @@ -346,7 +342,6 @@ public void testContainerLauncherUserError() throws ServicePluginException { ContainerLauncherLaunchRequestEvent launchRequestEvent = new ContainerLauncherLaunchRequestEvent(clc1, container1, 0, 0, 0); - containerLauncherManager.handle(launchRequestEvent); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Event.class); @@ -403,7 +398,6 @@ private static class ContainerLaucherRouterForMultipleLauncherTest private static final List containerLauncherNames = new LinkedList<>(); private static final List testContainerLaunchers = new LinkedList<>(); - public static void reset() { numContainerLaunchers.set(0); containerLauncherIndices.clear(); @@ -516,6 +510,7 @@ public void stopContainer(ContainerStopRequest stopRequest) { private static final String DAG_NAME = "dagName"; private static final int DAG_INDEX = 1; + public static class ContainerLauncherForTest extends ContainerLauncher { public ContainerLauncherForTest( @@ -535,5 +530,4 @@ public void stopContainer(ContainerStopRequest stopRequest) throws ServicePlugin .reportError(ServicePluginErrorDefaults.SERVICE_UNAVAILABLE, "ReportError", new DagInfoImplForTest(DAG_INDEX, DAG_NAME)); } } - } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestContainerLauncherWrapper.java b/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestContainerLauncherWrapper.java index cb7d62dca0..d484eb3a2c 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestContainerLauncherWrapper.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestContainerLauncherWrapper.java @@ -26,5 +26,4 @@ public void testDelegation() throws Exception { PluginWrapperTestHelpers.testDelegation(ContainerLauncherWrapper.class, ContainerLauncher.class, Sets.newHashSet("getContainerLauncher", "dagComplete", "vertexComplete", "taskAttemptFailed")); } - } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestDeletionTracker.java b/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestDeletionTracker.java index fba35a69f2..2f97c8a401 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestDeletionTracker.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestDeletionTracker.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestTezLocalCacheManager.java b/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestTezLocalCacheManager.java index 5596dc8feb..a662a1dace 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestTezLocalCacheManager.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/launcher/TestTezLocalCacheManager.java @@ -126,7 +126,6 @@ public void testLocalizeRootDirectory() throws URISyntaxException, IOException { Assert.assertFalse(Files.exists(Paths.get("./file-one"))); manager.localize(); Assert.assertTrue(Files.exists(Paths.get("./file-one"))); - } finally { manager.cleanup(); Assert.assertFalse(Files.exists(Paths.get("./file-one"))); @@ -145,7 +144,6 @@ public void testLocalizeRootDirectory() throws URISyntaxException, IOException { // file appears only at configured location Assert.assertFalse(Files.exists(Paths.get("./file-one"))); Assert.assertTrue(Files.exists(Paths.get("./target/file-one"))); - } finally { manager.cleanup(); Assert.assertFalse(Files.exists(Paths.get("./target/file-one"))); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java index fe5a888b56..52c06110ff 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -115,13 +115,13 @@ public void testDelayedReuseContainerBecomesAvailable() LOG.info("Test testDelayedReuseContainerBecomesAvailable"); Configuration conf = new Configuration(); conf.setBoolean( - TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true); + TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true); conf.setBoolean( TezConfiguration.TEZ_AM_CONTAINER_REUSE_RACK_FALLBACK_ENABLED, false); conf.setBoolean( TezConfiguration.TEZ_AM_CONTAINER_REUSE_NON_LOCAL_FALLBACK_ENABLED, false); conf.setLong( - TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 3000l); + TezConfiguration.TEZ_AM_CONTAINER_REUSE_LOCALITY_DELAY_ALLOCATION_MILLIS, 3000l); conf.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS, 0); conf.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS, 0); @@ -131,14 +131,14 @@ public void testDelayedReuseContainerBecomesAvailable() AMRMClient rmClientCore = new AMRMClientForTest(); TezAMRMClientAsync rmClient = - spy(new AMRMClientAsyncForTest(rmClientCore, 100)); + spy(new AMRMClientAsyncForTest(rmClientCore, 100)); AppContext appContext = mock(AppContext.class); doReturn(conf).when(appContext).getAMConf(); AMContainerMap amContainerMap = new AMContainerMap( - mock(ContainerHeartbeatHandler.class), - mock(TaskCommunicatorManagerInterface.class), - new ContainerContextMatcher(), appContext); + mock(ContainerHeartbeatHandler.class), + mock(TaskCommunicatorManagerInterface.class), + new ContainerContextMatcher(), appContext); AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext); doReturn(amContainerMap).when(appContext).getAllContainers(); doReturn(DAGAppMasterState.RUNNING).when(appContext).getAMState(); @@ -148,49 +148,49 @@ public void testDelayedReuseContainerBecomesAvailable() TaskSchedulerManager taskSchedulerManagerReal = new TaskSchedulerManagerForTest( - appContext, eventHandler, rmClient, - new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(conf)); + appContext, eventHandler, rmClient, + new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(conf)); TaskSchedulerManager taskSchedulerManager = spy(taskSchedulerManagerReal); taskSchedulerManager.init(conf); taskSchedulerManager.start(); TaskSchedulerWithDrainableContext taskScheduler = - (TaskSchedulerWithDrainableContext) - ((TaskSchedulerManagerForTest) taskSchedulerManager) - .getSpyTaskScheduler(); + (TaskSchedulerWithDrainableContext) + ((TaskSchedulerManagerForTest) taskSchedulerManager) + .getSpyTaskScheduler(); TaskSchedulerContextDrainable drainableAppCallback = - taskScheduler.getDrainableAppCallback(); + taskScheduler.getDrainableAppCallback(); AtomicBoolean drainNotifier = new AtomicBoolean(false); taskScheduler.delayedContainerManager.drainedDelayedContainersForTest = drainNotifier; Resource resource = Resource.newInstance(1024, 1); Priority priority = Priority.newInstance(5); - String [] host1 = {"host1"}; - String [] host2 = {"host2"}; + String[] host1 = {"host1"}; + String[] host2 = {"host2"}; - String [] defaultRack = {"/default-rack"}; + String[] defaultRack = {"/default-rack"}; TezTaskAttemptID taID11 = TezTaskAttemptID.getInstance( - TezTaskID.getInstance(vertexID, 1), 1); + TezTaskID.getInstance(vertexID, 1), 1); TezTaskAttemptID taID21 = TezTaskAttemptID.getInstance( - TezTaskID.getInstance(vertexID, 2), 1); + TezTaskID.getInstance(vertexID, 2), 1); TezTaskAttemptID taID31 = TezTaskAttemptID.getInstance( - TezTaskID.getInstance(vertexID, 3), 1); + TezTaskID.getInstance(vertexID, 3), 1); TaskAttempt ta11 = mock(TaskAttempt.class); TaskAttempt ta21 = mock(TaskAttempt.class); TaskAttempt ta31 = mock(TaskAttempt.class); AMSchedulerEventTALaunchRequest lrTa11 = - createLaunchRequestEvent(taID11, ta11, resource, host1, - defaultRack, priority); + createLaunchRequestEvent(taID11, ta11, resource, host1, + defaultRack, priority); AMSchedulerEventTALaunchRequest lrTa21 = - createLaunchRequestEvent(taID21, ta21, resource, host2, - defaultRack, priority); + createLaunchRequestEvent(taID21, ta21, resource, host2, + defaultRack, priority); AMSchedulerEventTALaunchRequest lrTa31 = - createLaunchRequestEvent(taID31, ta31, resource, host1, - defaultRack, priority); + createLaunchRequestEvent(taID31, ta31, resource, host1, + defaultRack, priority); taskSchedulerManager.handleEvent(lrTa11); taskSchedulerManager.handleEvent(lrTa21); @@ -199,7 +199,7 @@ public void testDelayedReuseContainerBecomesAvailable() Container containerHost2 = createContainer(2, host2[0], resource, priority); taskScheduler.onContainersAllocated( - Lists.newArrayList(containerHost1, containerHost2)); + Lists.newArrayList(containerHost1, containerHost2)); TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier); drainableAppCallback.drain(); verify(taskSchedulerManager).taskAllocated( @@ -224,15 +224,15 @@ public void testDelayedReuseContainerBecomesAvailable() eventHandler.reset(); taskSchedulerManager.handleEvent( - new AMSchedulerEventTAEnded(ta21, containerHost2.getId(), - TaskAttemptState.SUCCEEDED, null, null, 0)); + new AMSchedulerEventTAEnded(ta21, containerHost2.getId(), + TaskAttemptState.SUCCEEDED, null, null, 0)); long currentTs = System.currentTimeMillis(); Throwable exception = null; while (System.currentTimeMillis() < currentTs + 5000l) { try { verify(taskSchedulerManager, - times(1)).containerBeingReleased(eq(0), eq(containerHost2.getId())); + times(1)).containerBeingReleased(eq(0), eq(containerHost2.getId())); exception = null; break; } catch (Throwable e) { @@ -262,14 +262,14 @@ public void testDelayedReuseContainerNotAvailable() AMRMClient rmClientCore = new AMRMClientForTest(); TezAMRMClientAsync rmClient = - spy(new AMRMClientAsyncForTest(rmClientCore, 100)); + spy(new AMRMClientAsyncForTest(rmClientCore, 100)); AppContext appContext = mock(AppContext.class); doReturn(new Configuration(false)).when(appContext).getAMConf(); AMContainerMap amContainerMap = new AMContainerMap( - mock(ContainerHeartbeatHandler.class), - mock(TaskCommunicatorManagerInterface.class), - new ContainerContextMatcher(), appContext); + mock(ContainerHeartbeatHandler.class), + mock(TaskCommunicatorManagerInterface.class), + new ContainerContextMatcher(), appContext); AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext); doReturn(amContainerMap).when(appContext).getAllContainers(); doReturn(amNodeTracker).when(appContext).getNodeTracker(); @@ -278,29 +278,29 @@ public void testDelayedReuseContainerNotAvailable() doReturn(mock(ClusterInfo.class)).when(appContext).getClusterInfo(); TaskSchedulerManager taskSchedulerManagerReal = - new TaskSchedulerManagerForTest(appContext, eventHandler, rmClient, - new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(conf)); + new TaskSchedulerManagerForTest(appContext, eventHandler, rmClient, + new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(conf)); TaskSchedulerManager taskSchedulerManager = - spy(taskSchedulerManagerReal); + spy(taskSchedulerManagerReal); taskSchedulerManager.init(conf); taskSchedulerManager.start(); TaskSchedulerWithDrainableContext taskScheduler = - (TaskSchedulerWithDrainableContext) - ((TaskSchedulerManagerForTest) taskSchedulerManager) - .getSpyTaskScheduler(); + (TaskSchedulerWithDrainableContext) + ((TaskSchedulerManagerForTest) taskSchedulerManager) + .getSpyTaskScheduler(); TaskSchedulerContextDrainable drainableAppCallback = - taskScheduler.getDrainableAppCallback(); - + taskScheduler.getDrainableAppCallback(); + AtomicBoolean drainNotifier = new AtomicBoolean(false); taskScheduler.delayedContainerManager.drainedDelayedContainersForTest = drainNotifier; Resource resource = Resource.newInstance(1024, 1); Priority priority = Priority.newInstance(5); - String [] host1 = {"host1"}; - String [] host2 = {"host2"}; + String[] host1 = {"host1"}; + String[] host2 = {"host2"}; - String [] defaultRack = {"/default-rack"}; + String[] defaultRack = {"/default-rack"}; TezTaskAttemptID taID11 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID, 1), 1); TezTaskAttemptID taID21 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID, 2), 1); @@ -310,11 +310,11 @@ public void testDelayedReuseContainerNotAvailable() TaskAttempt ta31 = mock(TaskAttempt.class); AMSchedulerEventTALaunchRequest lrTa11 = createLaunchRequestEvent( - taID11, ta11, resource, host1, defaultRack, priority); + taID11, ta11, resource, host1, defaultRack, priority); AMSchedulerEventTALaunchRequest lrTa21 = createLaunchRequestEvent( - taID21, ta21, resource, host2, defaultRack, priority); + taID21, ta21, resource, host2, defaultRack, priority); AMSchedulerEventTALaunchRequest lrTa31 = createLaunchRequestEvent( - taID31, ta31, resource, host1, defaultRack, priority); + taID31, ta31, resource, host1, defaultRack, priority); taskSchedulerManager.handleEvent(lrTa11); taskSchedulerManager.handleEvent(lrTa21); @@ -340,7 +340,7 @@ public void testDelayedReuseContainerNotAvailable() verify(taskSchedulerManager, times(0)).taskAllocated( eq(0), eq(ta31), any(), eq(containerHost2)); verify(rmClient, times(1)).releaseAssignedContainer( - eq(containerHost2.getId())); + eq(containerHost2.getId())); eventHandler.verifyInvocation(AMContainerEventStopRequest.class); taskScheduler.shutdown(); @@ -390,7 +390,7 @@ public void testSimpleReuse() throws IOException, InterruptedException, Executio String[] host1 = {"host1"}; String[] host2 = {"host2"}; - String []racks = {"/default-rack"}; + String[] racks = {"/default-rack"}; Priority priority1 = Priority.newInstance(1); TezVertexID vertexID1 = TezVertexID.getInstance(dagID, 1); @@ -500,7 +500,7 @@ public void testReuseWithTaskSpecificLaunchCmdOption() throws IOException, Inter //Profile 3 tasks tezConf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST, "v1[1,3,4]"); tezConf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS, "dir=/tmp/__VERTEX_NAME__/__TASK_INDEX__"); - TaskSpecificLaunchCmdOption taskSpecificLaunchCmdOption = new TaskSpecificLaunchCmdOption(tezConf); + TaskSpecificLaunchCmdOption taskSpecificLaunchCmdOption = new TaskSpecificLaunchCmdOption(tezConf); CapturingEventHandler eventHandler = new CapturingEventHandler(); TezDAGID dagID = TezDAGID.getInstance("0", 0, 0); @@ -528,7 +528,7 @@ public void testReuseWithTaskSpecificLaunchCmdOption() throws IOException, Inter TaskSchedulerWithDrainableContext taskScheduler = (TaskSchedulerWithDrainableContext) ((TaskSchedulerManagerForTest) taskSchedulerManager) - .getSpyTaskScheduler(); + .getSpyTaskScheduler(); TaskSchedulerContextDrainable drainableAppCallback = taskScheduler.getDrainableAppCallback(); AtomicBoolean drainNotifier = new AtomicBoolean(false); taskScheduler.delayedContainerManager.drainedDelayedContainersForTest = drainNotifier; @@ -538,7 +538,7 @@ public void testReuseWithTaskSpecificLaunchCmdOption() throws IOException, Inter String[] host2 = {"host2"}; String[] host3 = {"host3"}; - String []racks = {"/default-rack"}; + String[] racks = {"/default-rack"}; Priority priority1 = Priority.newInstance(1); TezVertexID vertexID1 = TezVertexID.getInstance(dagID, 1); @@ -554,7 +554,7 @@ public void testReuseWithTaskSpecificLaunchCmdOption() throws IOException, Inter TaskAttempt ta11 = mock(TaskAttempt.class); AMSchedulerEventTALaunchRequest lrEvent1 = createLaunchRequestEvent(taID11, ta11, resource1, host1, racks, - priority1, localResources, tsLaunchCmdOpts); + priority1, localResources, tsLaunchCmdOpts); //Vertex 1, Task 2, Attempt 1, host1 TezTaskAttemptID taID12 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID1, 2), 1); @@ -635,7 +635,7 @@ public void testReuseWithTaskSpecificLaunchCmdOption() throws IOException, Inter */ tezConf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST, "v1[1,2,3,5,6]"); tezConf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS, "dummyOpts"); - taskSpecificLaunchCmdOption = new TaskSpecificLaunchCmdOption(tezConf); + taskSpecificLaunchCmdOption = new TaskSpecificLaunchCmdOption(tezConf); //Vertex 1, Task 5, Attempt 1, host3 TezTaskAttemptID taID15 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID1, 3), 1); @@ -651,7 +651,7 @@ public void testReuseWithTaskSpecificLaunchCmdOption() throws IOException, Inter TaskAttempt ta16 = mock(TaskAttempt.class); AMSchedulerEventTALaunchRequest lrEvent6 = createLaunchRequestEvent(taID16, ta16, resource1, host3, racks, - priority1, localResources, taskSpecificLaunchCmdOption.getTaskSpecificOption("", "v1", 6)); + priority1, localResources, taskSpecificLaunchCmdOption.getTaskSpecificOption("", "v1", 6)); // Container started Container container3 = createContainer(3, "host3", resource1, priority1); @@ -711,25 +711,25 @@ public void testReuseNonLocalRequest() TaskSchedulerManager taskSchedulerManagerReal = new TaskSchedulerManagerForTest( - appContext, eventHandler, rmClient, - new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(tezConf)); + appContext, eventHandler, rmClient, + new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(tezConf)); TaskSchedulerManager taskSchedulerManager = spy(taskSchedulerManagerReal); taskSchedulerManager.init(tezConf); taskSchedulerManager.start(); TaskSchedulerWithDrainableContext taskScheduler = - (TaskSchedulerWithDrainableContext) - ((TaskSchedulerManagerForTest) taskSchedulerManager) - .getSpyTaskScheduler(); + (TaskSchedulerWithDrainableContext) + ((TaskSchedulerManagerForTest) taskSchedulerManager) + .getSpyTaskScheduler(); TaskSchedulerContextDrainable drainableAppCallback = - taskScheduler.getDrainableAppCallback(); + taskScheduler.getDrainableAppCallback(); AtomicBoolean drainNotifier = new AtomicBoolean(false); taskScheduler.delayedContainerManager.drainedDelayedContainersForTest = drainNotifier; Resource resource1 = Resource.newInstance(1024, 1); - String [] emptyHosts = new String[0]; - String [] racks = { "default-rack" }; + String[] emptyHosts = new String[0]; + String[] racks = {"default-rack"}; Priority priority = Priority.newInstance(3); @@ -740,14 +740,14 @@ public void testReuseNonLocalRequest() TaskAttempt ta11 = mock(TaskAttempt.class); doReturn(vertexID).when(ta11).getVertexID(); AMSchedulerEventTALaunchRequest lrEvent11 = createLaunchRequestEvent( - taID11, ta11, resource1, emptyHosts, racks, priority); + taID11, ta11, resource1, emptyHosts, racks, priority); //Vertex1, Task2, Attempt 1, no locality information. TezTaskAttemptID taID12 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID, 2), 1); TaskAttempt ta12 = mock(TaskAttempt.class); doReturn(vertexID).when(ta12).getVertexID(); AMSchedulerEventTALaunchRequest lrEvent12 = createLaunchRequestEvent( - taID12, ta12, resource1, emptyHosts, racks, priority); + taID12, ta12, resource1, emptyHosts, racks, priority); // Send launch request for task 1 only, deterministic assignment to this task. taskSchedulerManager.handleEvent(lrEvent11); @@ -785,8 +785,8 @@ public void testReuseNonLocalRequest() // TA12 completed. taskSchedulerManager.handleEvent( - new AMSchedulerEventTAEnded(ta12, container1.getId(), - TaskAttemptState.SUCCEEDED, null, null, 0)); + new AMSchedulerEventTAEnded(ta12, container1.getId(), + TaskAttemptState.SUCCEEDED, null, null, 0)); TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier); drainableAppCallback.drain(); @@ -809,7 +809,7 @@ public void testReuseAcrossVertices() tezConf.setLong( TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS, 20l); tezConf.setLong( - TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS, 30l); + TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS, 30l); tezConf.setInt( TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, 1); @@ -818,14 +818,14 @@ public void testReuseAcrossVertices() AMRMClient rmClientCore = new AMRMClientForTest(); TezAMRMClientAsync rmClient = - spy(new AMRMClientAsyncForTest(rmClientCore, 100)); + spy(new AMRMClientAsyncForTest(rmClientCore, 100)); AppContext appContext = mock(AppContext.class); doReturn(new Configuration(false)).when(appContext).getAMConf(); AMContainerMap amContainerMap = new AMContainerMap( - mock(ContainerHeartbeatHandler.class), - mock(TaskCommunicatorManagerInterface.class), - new ContainerContextMatcher(), appContext); + mock(ContainerHeartbeatHandler.class), + mock(TaskCommunicatorManagerInterface.class), + new ContainerContextMatcher(), appContext); AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext); doReturn(amContainerMap).when(appContext).getAllContainers(); doReturn(amNodeTracker).when(appContext).getNodeTracker(); @@ -835,17 +835,17 @@ public void testReuseAcrossVertices() doReturn(mock(ClusterInfo.class)).when(appContext).getClusterInfo(); TaskSchedulerManager taskSchedulerManagerReal = - new TaskSchedulerManagerForTest(appContext, eventHandler, rmClient, - new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(tezConf)); + new TaskSchedulerManagerForTest(appContext, eventHandler, rmClient, + new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(tezConf)); TaskSchedulerManager taskSchedulerManager = - spy(taskSchedulerManagerReal); + spy(taskSchedulerManagerReal); taskSchedulerManager.init(tezConf); taskSchedulerManager.start(); TaskSchedulerWithDrainableContext taskScheduler = - (TaskSchedulerWithDrainableContext) - ((TaskSchedulerManagerForTest) taskSchedulerManager) - .getSpyTaskScheduler(); + (TaskSchedulerWithDrainableContext) + ((TaskSchedulerManagerForTest) taskSchedulerManager) + .getSpyTaskScheduler(); TaskSchedulerContextDrainable drainableAppCallback = taskScheduler.getDrainableAppCallback(); AtomicBoolean drainNotifier = new AtomicBoolean(false); @@ -854,7 +854,7 @@ public void testReuseAcrossVertices() Resource resource1 = Resource.newInstance(1024, 1); String[] host1 = {"host1"}; - String []racks = {"/default-rack"}; + String[] racks = {"/default-rack"}; Priority priority1 = Priority.newInstance(3); Priority priority2 = Priority.newInstance(4); @@ -863,7 +863,7 @@ public void testReuseAcrossVertices() //Vertex 1, Task 1, Attempt 1, host1 TezTaskAttemptID taID11 = TezTaskAttemptID.getInstance( - TezTaskID.getInstance(vertexID1, 1), 1); + TezTaskID.getInstance(vertexID1, 1), 1); TaskAttempt ta11 = mock(TaskAttempt.class); doReturn(vertexID1).when(ta11).getVertexID(); AMSchedulerEventTALaunchRequest lrEvent11 = createLaunchRequestEvent( @@ -871,11 +871,11 @@ public void testReuseAcrossVertices() //Vertex2, Task1, Attempt 1, host1 TezTaskAttemptID taID21 = TezTaskAttemptID.getInstance( - TezTaskID.getInstance(vertexID2, 1), 1); + TezTaskID.getInstance(vertexID2, 1), 1); TaskAttempt ta21 = mock(TaskAttempt.class); doReturn(vertexID2).when(ta21).getVertexID(); AMSchedulerEventTALaunchRequest lrEvent21 = createLaunchRequestEvent( - taID21, ta21, resource1, host1, racks, priority2); + taID21, ta21, resource1, host1, racks, priority2); // Send launch request for task 1 onle, deterministic assignment to this task. taskSchedulerManager.handleEvent(lrEvent11); @@ -918,7 +918,7 @@ public void testReuseAcrossVertices() taskScheduler.shutdown(); taskSchedulerManager.close(); } - + @Test(timeout = 30000l) public void testReuseLocalResourcesChanged() throws IOException, InterruptedException, ExecutionException { LOG.info("Test testReuseLocalResourcesChanged"); @@ -947,7 +947,7 @@ public void testReuseLocalResourcesChanged() throws IOException, InterruptedExce doReturn(true).when(appContext).isSession(); doAnswer(dagIDAnswer).when(appContext).getCurrentDAGID(); doReturn(mock(ClusterInfo.class)).when(appContext).getClusterInfo(); - + TaskSchedulerManager taskSchedulerManagerReal = new TaskSchedulerManagerForTest(appContext, eventHandler, rmClient, new AlwaysMatchesContainerMatcher(), TezUtils.createUserPayloadFromConf(tezConf)); TaskSchedulerManager taskSchedulerManager = spy(taskSchedulerManagerReal); @@ -963,7 +963,7 @@ public void testReuseLocalResourcesChanged() throws IOException, InterruptedExce Resource resource1 = Resource.newInstance(1024, 1); String[] host1 = {"host1"}; - String []racks = {"/default-rack"}; + String[] racks = {"/default-rack"}; Priority priority1 = Priority.newInstance(1); String rsrc1 = "rsrc1"; @@ -974,12 +974,12 @@ public void testReuseLocalResourcesChanged() throws IOException, InterruptedExce LocalResource lr3 = mock(LocalResource.class); AMContainerEventAssignTA assignEvent = null; - + Map dag1LRs = Maps.newHashMap(); dag1LRs.put(rsrc1, lr1); TezVertexID vertexID11 = TezVertexID.getInstance(dagID1, 1); - + //Vertex 1, Task 1, Attempt 1, host1, lr1 TezTaskAttemptID taID111 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID11, 1), 1); TaskAttempt ta111 = mock(TaskAttempt.class); @@ -1002,7 +1002,7 @@ public void testReuseLocalResourcesChanged() throws IOException, InterruptedExce verify(taskSchedulerManager).taskAllocated(eq(0), eq(ta111), any(), eq(container1)); assignEvent = (AMContainerEventAssignTA) eventHandler.verifyInvocation(AMContainerEventAssignTA.class); assertEquals(1, assignEvent.getRemoteTaskLocalResources().size()); - + // Task assigned to container completed successfully. Container should be re-used. taskSchedulerManager.handleEvent( new AMSchedulerEventTAEnded(ta111, container1.getId(), TaskAttemptState.SUCCEEDED, null, @@ -1030,13 +1030,13 @@ public void testReuseLocalResourcesChanged() throws IOException, InterruptedExce // Setup DAG2 with additional resources. Make sure the container, even without all resources, is reused. TezDAGID dagID2 = TezDAGID.getInstance("0", 2, 0); dagIDAnswer.setDAGID(dagID2); - + Map dag2LRs = Maps.newHashMap(); dag2LRs.put(rsrc2, lr2); dag2LRs.put(rsrc3, lr3); TezVertexID vertexID21 = TezVertexID.getInstance(dagID2, 1); - + //Vertex 2, Task 1, Attempt 1, host1, lr2 TezTaskAttemptID taID211 = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexID21, 1), 1); TaskAttempt ta211 = mock(TaskAttempt.class); @@ -1121,7 +1121,7 @@ public void testReuseConflictLocalResources() throws IOException, InterruptedExc Resource resource1 = Resource.newInstance(1024, 1); String[] host1 = {"host1"}; - String []racks = {"/default-rack"}; + String[] racks = {"/default-rack"}; Priority priority1 = Priority.newInstance(1); String rsrc1 = "rsrc1"; @@ -1175,7 +1175,6 @@ public void testReuseConflictLocalResources() throws IOException, InterruptedExc AMSchedulerEventTALaunchRequest lrEvent14 = createLaunchRequestEvent( taID114, ta114, resource1, host1, racks, priority1, new HashMap()); - taskSchedulerManager.handleEvent(lrEvent11); taskSchedulerManager.handleEvent(lrEvent12); @@ -1247,7 +1246,6 @@ public void testReuseConflictLocalResources() throws IOException, InterruptedExc eventHandler.verifyNoInvocations(AMContainerEventStopRequest.class); eventHandler.reset(); - // Setup DAG2 with different resources. TezDAGID dagID2 = TezDAGID.getInstance("0", 2, 0); dagIDAnswer.setDAGID(dagID2); @@ -1286,7 +1284,6 @@ public void testReuseConflictLocalResources() throws IOException, InterruptedExc verify(rmClient, times(0)).releaseAssignedContainer(eq(container2.getId())); eventHandler.reset(); - // Setup DAG3 with a subset of resources from DAG2 (container should be reused). TezDAGID dagID3 = TezDAGID.getInstance("0", 3, 0); dagIDAnswer.setDAGID(dagID3); @@ -1350,7 +1347,7 @@ public void testAssignmentOnShutdown() TaskSchedulerWithDrainableContext taskScheduler = (TaskSchedulerWithDrainableContext) ((TaskSchedulerManagerForTest) taskSchedulerManager) - .getSpyTaskScheduler(); + .getSpyTaskScheduler(); TaskSchedulerContextDrainable drainableAppCallback = taskScheduler.getDrainableAppCallback(); AtomicBoolean drainNotifier = new AtomicBoolean(false); taskScheduler.delayedContainerManager.drainedDelayedContainersForTest = drainNotifier; @@ -1358,7 +1355,7 @@ public void testAssignmentOnShutdown() Resource resource1 = Resource.newInstance(1024, 1); String[] host1 = {"host1"}; - String []racks = {"/default-rack"}; + String[] racks = {"/default-rack"}; Priority priority1 = Priority.newInstance(1); TezVertexID vertexID1 = TezVertexID.getInstance(dagID, 1); @@ -1381,7 +1378,7 @@ public void testAssignmentOnShutdown() taskSchedulerManager.close(); } - @Test(timeout=5000) + @Test(timeout = 5000) public void testDifferentResourceContainerReuse() throws Exception { Configuration tezConf = new Configuration(); tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true); @@ -1426,7 +1423,7 @@ eventHandler, rmClient, new AlwaysMatchesContainerMatcher(), String[] host1 = {"host1"}; String[] host2 = {"host2"}; - String []racks = {"/default-rack"}; + String[] racks = {"/default-rack"}; Priority priority1 = Priority.newInstance(1); TezVertexID vertexID1 = TezVertexID.getInstance(dagID, 1); @@ -1517,7 +1514,7 @@ eventHandler, rmClient, new AlwaysMatchesContainerMatcher(), taskSchedulerManager.close(); } - @Test(timeout=5000) + @Test(timeout = 5000) public void testEnvironmentVarsContainerReuse() throws Exception { Configuration tezConf = new Configuration(); tezConf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, true); @@ -1560,7 +1557,7 @@ eventHandler, rmClient, new ContainerContextMatcher(), Resource resource1 = Resource.newInstance(1024, 1); String[] host1 = {"host1"}; - String []racks = {"/default-rack"}; + String[] racks = {"/default-rack"}; Priority priority1 = Priority.newInstance(1); TezVertexID vertexID1 = TezVertexID.getInstance(dagID, 1); @@ -1676,9 +1673,9 @@ private Container createContainer(int id, String host, Resource resource, Priori } private AMSchedulerEventTALaunchRequest createLaunchRequestEvent( - TezTaskAttemptID taID, TaskAttempt ta, Resource capability, - String[] hosts, String[] racks, Priority priority, - ContainerContext containerContext) { + TezTaskAttemptID taID, TaskAttempt ta, Resource capability, + String[] hosts, String[] racks, Priority priority, + ContainerContext containerContext) { TaskLocationHint locationHint = null; if (hosts != null || racks != null) { Set hostsSet = Sets.newHashSet(hosts); @@ -1686,39 +1683,39 @@ private AMSchedulerEventTALaunchRequest createLaunchRequestEvent( locationHint = TaskLocationHint.createTaskLocationHint(hostsSet, racksSet); } AMSchedulerEventTALaunchRequest lr = new AMSchedulerEventTALaunchRequest( - taID, capability, new TaskSpec(taID, "dagName", "vertexName", -1, + taID, capability, new TaskSpec(taID, "dagName", "vertexName", -1, ProcessorDescriptor.create("processorClassName"), - Collections.singletonList(new InputSpec("vertexName", - InputDescriptor.create("inputClassName"), 1)), - Collections.singletonList(new OutputSpec("vertexName", - OutputDescriptor.create("outputClassName"), 1)), null, null), ta, locationHint, - priority.getPriority(), containerContext, 0, 0, 0); + Collections.singletonList(new InputSpec("vertexName", + InputDescriptor.create("inputClassName"), 1)), + Collections.singletonList(new OutputSpec("vertexName", + OutputDescriptor.create("outputClassName"), 1)), null, null), ta, locationHint, + priority.getPriority(), containerContext, 0, 0, 0); return lr; } private AMSchedulerEventTALaunchRequest createLaunchRequestEvent(TezTaskAttemptID taID, - TaskAttempt ta, Resource capability, String[] hosts, String[] racks, Priority priority) { + TaskAttempt ta, Resource capability, String[] hosts, String[] racks, Priority priority) { return createLaunchRequestEvent(taID, ta, capability, hosts, racks, priority, new HashMap()); } private AMSchedulerEventTALaunchRequest createLaunchRequestEvent(TezTaskAttemptID taID, - TaskAttempt ta, Resource capability, String[] hosts, String[] racks, Priority priority, - Map localResources) { + TaskAttempt ta, Resource capability, String[] hosts, String[] racks, Priority priority, + Map localResources) { return createLaunchRequestEvent(taID, ta, capability, hosts, racks, priority, localResources, ""); } private AMSchedulerEventTALaunchRequest createLaunchRequestEvent(TezTaskAttemptID taID, - TaskAttempt ta, Resource capability, String[] hosts, String[] racks, Priority priority, - Map localResources, String jvmOpts) { + TaskAttempt ta, Resource capability, String[] hosts, String[] racks, Priority priority, + Map localResources, String jvmOpts) { return createLaunchRequestEvent(taID, ta, capability, hosts, racks, priority, new ContainerContext(localResources, new Credentials(), new HashMap(), jvmOpts)); } private AMSchedulerEventTALaunchRequest createLaunchRequestEvent(TezTaskAttemptID taID, - TaskAttempt ta, Resource capability, String[] hosts, String[] racks, Priority priority, - String jvmOpts, Map environment) { + TaskAttempt ta, Resource capability, String[] hosts, String[] racks, Priority priority, + String jvmOpts, Map environment) { return createLaunchRequestEvent(taID, ta, capability, hosts, racks, priority, new ContainerContext(new HashMap(), new Credentials(), environment, jvmOpts)); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestDagAwareYarnTaskScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestDagAwareYarnTaskScheduler.java index 1465bfaaf6..a615183544 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestDagAwareYarnTaskScheduler.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestDagAwareYarnTaskScheduler.java @@ -119,8 +119,8 @@ private TaskSchedulerContextDrainable createDrainableContext( return new TaskSchedulerContextDrainable(wrapper); } - @SuppressWarnings({ "unchecked" }) - @Test(timeout=30000) + @SuppressWarnings({"unchecked"}) + @Test(timeout = 30000) public void testNoReuse() throws Exception { AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest()); @@ -396,7 +396,7 @@ public void testNoReuse() throws Exception { verify(mockRMClient).stop(); } - @Test(timeout=30000) + @Test(timeout = 30000) public void testSimpleReuseLocalMatching() throws Exception { AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest()); @@ -438,7 +438,7 @@ public void testSimpleReuseLocalMatching() throws Exception { Priority priorityv0 = Priority.newInstance(1); Priority priorityv1 = Priority.newInstance(2); - String[] hostsv0t0 = { "host1", "host2" }; + String[] hostsv0t0 = {"host1", "host2"}; MockTaskInfo taskv0t0 = new MockTaskInfo("taskv0t0", priorityv0, hostsv0t0); MockTaskInfo taskv0t1 = new MockTaskInfo("taskv0t1", priorityv0, "host3"); MockTaskInfo taskv0t2 = new MockTaskInfo("taskv0t2", priorityv0, hostsv0t0); @@ -501,7 +501,7 @@ public void testSimpleReuseLocalMatching() throws Exception { verify(mockRMClient).stop(); } - @Test(timeout=30000) + @Test(timeout = 30000) public void testSimpleReuseRackMatching() throws Exception { AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest()); @@ -543,7 +543,7 @@ public void testSimpleReuseRackMatching() throws Exception { Priority priorityv0 = Priority.newInstance(1); Priority priorityv1 = Priority.newInstance(2); - String[] hostsv0t0 = { "host1", "host2" }; + String[] hostsv0t0 = {"host1", "host2"}; MockTaskInfo taskv0t0 = new MockTaskInfo("taskv0t0", priorityv0, hostsv0t0); MockTaskInfo taskv0t1 = new MockTaskInfo("taskv0t1", priorityv0, "host2"); MockTaskInfo taskv0t2 = new MockTaskInfo("taskv0t2", priorityv0, "host4", "/somerack"); @@ -608,7 +608,7 @@ public void testSimpleReuseRackMatching() throws Exception { verify(mockRMClient).stop(); } - @Test(timeout=30000) + @Test(timeout = 30000) public void testSimpleReuseAnyMatching() throws Exception { AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest()); @@ -650,7 +650,7 @@ public void testSimpleReuseAnyMatching() throws Exception { Priority priorityv0 = Priority.newInstance(1); Priority priorityv1 = Priority.newInstance(2); - String[] hostsv0t0 = { "host1", "host2" }; + String[] hostsv0t0 = {"host1", "host2"}; MockTaskInfo taskv0t0 = new MockTaskInfo("taskv0t0", priorityv0, hostsv0t0); MockTaskInfo taskv0t1 = new MockTaskInfo("taskv0t1", priorityv0, "host2"); MockTaskInfo taskv0t2 = new MockTaskInfo("taskv0t2", priorityv0, "host4", "/rack4"); @@ -724,7 +724,7 @@ public void testSimpleReuseAnyMatching() throws Exception { verify(mockRMClient).stop(); } - @Test(timeout=30000) + @Test(timeout = 30000) public void testReuseWithAffinity() throws Exception { AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest()); @@ -766,7 +766,7 @@ public void testReuseWithAffinity() throws Exception { Priority priorityv0 = Priority.newInstance(1); Priority priorityv1 = Priority.newInstance(2); - String[] hostsv0t0 = { "host1", "host2" }; + String[] hostsv0t0 = {"host1", "host2"}; MockTaskInfo taskv0t0 = new MockTaskInfo("taskv0t0", priorityv0, hostsv0t0); MockTaskInfo taskv0t1 = new MockTaskInfo("taskv0t1", priorityv0, hostsv0t0); @@ -812,7 +812,7 @@ public void testReuseWithAffinity() throws Exception { verify(mockRMClient).stop(); } - @Test(timeout=30000) + @Test(timeout = 30000) public void testReuseVertexDescendants() throws Exception { AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest()); @@ -830,7 +830,7 @@ public void testReuseVertexDescendants() throws Exception { // vertex 0 and vertex 2 are root vertices and vertex 1 is a child of vertex 0 DagInfo mockDagInfo = mock(DagInfo.class); when(mockDagInfo.getTotalVertices()).thenReturn(3); - when(mockDagInfo.getVertexDescendants(0)).thenReturn(BitSet.valueOf(new long[] { 0x2 })); + when(mockDagInfo.getVertexDescendants(0)).thenReturn(BitSet.valueOf(new long[]{0x2})); when(mockDagInfo.getVertexDescendants(1)).thenReturn(new BitSet()); when(mockDagInfo.getVertexDescendants(2)).thenReturn(new BitSet()); TaskSchedulerContext mockApp = setupMockTaskSchedulerContext(appHost, appPort, appUrl, conf); @@ -858,7 +858,7 @@ public void testReuseVertexDescendants() throws Exception { Priority priorityv0 = Priority.newInstance(1); Priority priorityv1 = Priority.newInstance(2); Priority priorityv2 = Priority.newInstance(3); - String[] hostsv0t0 = { "host1", "host2" }; + String[] hostsv0t0 = {"host1", "host2"}; MockTaskInfo taskv0t0 = new MockTaskInfo("taskv0t0", priorityv0, hostsv0t0); when(mockApp.getVertexIndexForTask(taskv0t0.task)).thenReturn(0); MockTaskInfo taskv0t1 = new MockTaskInfo("taskv0t1", priorityv0, "host3"); @@ -938,7 +938,7 @@ public void testReuseVertexDescendants() throws Exception { verify(mockRMClient).stop(); } - @Test(timeout=30000) + @Test(timeout = 30000) public void testSessionContainers() throws Exception { AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest()); @@ -1108,7 +1108,7 @@ public void testSessionContainers() throws Exception { verify(mockRMClient).stop(); } - @Test(timeout=50000) + @Test(timeout = 50000) public void testPreemptionNoHeadroom() throws Exception { AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest()); @@ -1129,7 +1129,7 @@ public void testPreemptionNoHeadroom() throws Exception { // vertex 0 and vertex 2 are root vertices and vertex 1 is a child of vertex 0 DagInfo mockDagInfo = mock(DagInfo.class); when(mockDagInfo.getTotalVertices()).thenReturn(3); - when(mockDagInfo.getVertexDescendants(0)).thenReturn(BitSet.valueOf(new long[] { 0x2 })); + when(mockDagInfo.getVertexDescendants(0)).thenReturn(BitSet.valueOf(new long[]{0x2})); when(mockDagInfo.getVertexDescendants(1)).thenReturn(new BitSet()); when(mockDagInfo.getVertexDescendants(2)).thenReturn(new BitSet()); TaskSchedulerContext mockApp = setupMockTaskSchedulerContext(appHost, appPort, appUrl, conf); @@ -1157,7 +1157,7 @@ public void testPreemptionNoHeadroom() throws Exception { Priority priorityv0 = Priority.newInstance(1); Priority priorityv1 = Priority.newInstance(2); Priority priorityv2 = Priority.newInstance(3); - String[] hostsv0t0 = { "host1", "host2" }; + String[] hostsv0t0 = {"host1", "host2"}; MockTaskInfo taskv0t0 = new MockTaskInfo("taskv0t0", priorityv0, hostsv0t0); when(mockApp.getVertexIndexForTask(taskv0t0.task)).thenReturn(0); MockTaskInfo taskv0t1 = new MockTaskInfo("taskv0t1", priorityv0, hostsv0t0); @@ -1251,7 +1251,7 @@ public void testPreemptionNoHeadroom() throws Exception { verify(mockRMClient).stop(); } - @Test (timeout = 50000L) + @Test(timeout = 50000L) public void testPreemptionWhenBlocked() throws Exception { AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest()); @@ -1269,11 +1269,10 @@ public void testPreemptionWhenBlocked() throws Exception { conf.setInt(TezConfiguration.TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS, 3); conf.setInt(TezConfiguration.TEZ_AM_PREEMPTION_MAX_WAIT_TIME_MS, 60 * 1000); - DagInfo mockDagInfo = mock(DagInfo.class); when(mockDagInfo.getTotalVertices()).thenReturn(3); - when(mockDagInfo.getVertexDescendants(0)).thenReturn(BitSet.valueOf(new long[] { 0x6 })); - when(mockDagInfo.getVertexDescendants(1)).thenReturn(BitSet.valueOf(new long[] { 0x2 })); + when(mockDagInfo.getVertexDescendants(0)).thenReturn(BitSet.valueOf(new long[]{0x6})); + when(mockDagInfo.getVertexDescendants(1)).thenReturn(BitSet.valueOf(new long[]{0x2})); when(mockDagInfo.getVertexDescendants(2)).thenReturn(new BitSet()); TaskSchedulerContext mockApp = setupMockTaskSchedulerContext(appHost, appPort, appUrl, conf); when(mockApp.getCurrentDagInfo()).thenReturn(mockDagInfo); @@ -1299,7 +1298,7 @@ public void testPreemptionWhenBlocked() throws Exception { Priority priorityv0 = Priority.newInstance(1); Priority priorityv2 = Priority.newInstance(3); - String[] hostsv0t0 = { "host1", "host2" }; + String[] hostsv0t0 = {"host1", "host2"}; MockTaskInfo taskv0t0 = new MockTaskInfo("taskv0t0", priorityv0, hostsv0t0); when(mockApp.getVertexIndexForTask(taskv0t0.task)).thenReturn(0); MockTaskInfo taskv0t1 = new MockTaskInfo("taskv0t1", priorityv0, hostsv0t0); @@ -1360,7 +1359,7 @@ public void testPreemptionWhenBlocked() throws Exception { verify(mockRMClient).stop(); } - @Test(timeout=50000) + @Test(timeout = 50000) public void testContainerAssignmentReleaseNewContainers() throws Exception { AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest()); @@ -1430,7 +1429,7 @@ public void testContainerAssignmentReleaseNewContainers() throws Exception { verify(mockRMClient).releaseAssignedContainer(eq(cid1)); } - @Test(timeout=50000) + @Test(timeout = 50000) public void testIdleContainerAssignmentReuseNewContainers() throws Exception { AMRMClientAsyncWrapperForTest mockRMClient = spy(new AMRMClientAsyncWrapperForTest()); @@ -1562,7 +1561,7 @@ RegisterApplicationMasterResponse getRegistrationResponse() { @Override public RegisterApplicationMasterResponse registerApplicationMaster(String appHostName, int appHostPort, - String appTrackingUrl) throws YarnException, IOException { + String appTrackingUrl) throws YarnException, IOException { return client.registerApplicationMaster(appHostName, appHostPort, appTrackingUrl); } @@ -1594,9 +1593,9 @@ protected void serviceStop() { @Override public RegisterApplicationMasterResponse registerApplicationMaster(String appHostName, int appHostPort, - String appTrackingUrl) { + String appTrackingUrl) { mockRegResponse = mock(RegisterApplicationMasterResponse.class); - Resource mockMaxResource = Resources.createResource(1024*1024, 1024); + Resource mockMaxResource = Resources.createResource(1024 * 1024, 1024); Map mockAcls = Collections.emptyMap(); when(mockRegResponse.getMaximumResourceCapability()).thenReturn( mockMaxResource); @@ -1608,7 +1607,7 @@ public RegisterApplicationMasterResponse registerApplicationMaster(String appHos @Override public void unregisterApplicationMaster(FinalApplicationStatus appStatus, - String appMessage, String appTrackingUrl) { + String appMessage, String appTrackingUrl) { } RegisterApplicationMasterResponse getRegistrationResponse() { @@ -1641,7 +1640,7 @@ static class MockTaskInfo { final Resource capability; MockTaskInfo(String name, Priority priority, String host) { - this(name, priority, host == null ? null : new String[] { host }); + this(name, priority, host == null ? null : new String[]{host}); } MockTaskInfo(String name, Priority priority, String[] hosts) { @@ -1649,8 +1648,8 @@ static class MockTaskInfo { } MockTaskInfo(String name, Priority priority, String host, String rack) { - this(name, priority, host == null ? null : new String[] { host }, - rack == null ? null : new String[] { rack }); + this(name, priority, host == null ? null : new String[]{host}, + rack == null ? null : new String[]{rack}); } MockTaskInfo(String name, Priority priority, String[] hosts, String[] racks) { @@ -1679,7 +1678,7 @@ static class TaskRequestCaptor { int invocationCount = 0; TaskRequestCaptor(AMRMClientAsync client, TaskScheduler scheduler, - TaskSchedulerContextDrainable drainableAppCallback) { + TaskSchedulerContextDrainable drainableAppCallback) { this.client = client; this.scheduler = scheduler; this.drainableAppCallback = drainableAppCallback; diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskScheduler.java index d7b516add2..cffad5623b 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskScheduler.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskScheduler.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -39,7 +39,6 @@ public class TestLocalTaskScheduler { - @Test(timeout = 5000) public void maxTasksAllocationsCannotBeExceeded() { @@ -61,11 +60,11 @@ public void maxTasksAllocationsCannotBeExceeded() { // Object under test AsyncDelegateRequestHandler requestHandler = - new AsyncDelegateRequestHandler(clientRequestQueue, - containerFactory, - taskAllocations, - mockContext, - tezConf); + new AsyncDelegateRequestHandler(clientRequestQueue, + containerFactory, + taskAllocations, + mockContext, + tezConf); // Allocate up to max tasks for (int i = 0; i < MAX_TASKS; i++) { diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskSchedulerService.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskSchedulerService.java index e193ee98f2..fc4af3ff13 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskSchedulerService.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestLocalTaskSchedulerService.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.app.rm; @@ -43,16 +43,16 @@ public class TestLocalTaskSchedulerService { - LocalTaskSchedulerService ltss ; - int core =10; + LocalTaskSchedulerService ltss; + int core = 10; @Test(timeout = 5000) public void testCreateResource() { Resource resource; //value in integer - long value = 4*1024*1024; - resource = ltss.createResource(value,core); - Assert.assertEquals((int)(value/(1024*1024)),resource.getMemory()); + long value = 4 * 1024 * 1024; + resource = ltss.createResource(value, core); + Assert.assertEquals((int) (value / (1024 * 1024)), resource.getMemory()); } @Test(timeout = 5000) @@ -73,7 +73,7 @@ public void testCreateResourceWithNegativeValue() { // it will be negative after it is passed to createResource try { - ltss.createResource((Long.MAX_VALUE*1024*1024), core); + ltss.createResource((Long.MAX_VALUE * 1024 * 1024), core); fail("No exception thrown."); } catch (Exception ex) { assertTrue(ex instanceof IllegalArgumentException); @@ -279,7 +279,7 @@ void allocateTask() { } public void drainRequest(int count) { - while(dispatchCount != count || !clientRequestQueue.isEmpty()) { + while (dispatchCount != count || !clientRequestQueue.isEmpty()) { try { Thread.sleep(100); } catch (InterruptedException e) { diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java index ff34084d37..bc1ec513b6 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -121,8 +121,8 @@ private TaskSchedulerContextDrainable createDrainableContext( return new TaskSchedulerContextDrainable(wrapper); } - @SuppressWarnings({ "unchecked" }) - @Test(timeout=10000) + @SuppressWarnings({"unchecked"}) + @Test(timeout = 10000) public void testTaskSchedulerNoReuse() throws Exception { AMRMClientAsyncForTest mockRMClient = spy( new AMRMClientAsyncForTest(new AMRMClientForTest(), 100)); @@ -150,9 +150,9 @@ public void testTaskSchedulerNoReuse() throws Exception { verify(mockRMClient).registerApplicationMaster(DEFAULT_APP_HOST, DEFAULT_APP_PORT, DEFAULT_APP_URL); RegisterApplicationMasterResponse regResponse = mockRMClient.getRegistrationResponse(); verify(mockApp).setApplicationRegistrationData(regResponse.getMaximumResourceCapability(), - regResponse.getApplicationACLs(), - regResponse.getClientToAMTokenMasterKey(), - regResponse.getQueue()); + regResponse.getApplicationACLs(), + regResponse.getClientToAMTokenMasterKey(), + regResponse.getQueue()); Assert.assertEquals(scheduler.getClusterNodeCount(), mockRMClient.getClusterNodeCount()); @@ -163,29 +163,29 @@ public void testTaskSchedulerNoReuse() throws Exception { String[] racks = {"/default-rack", "/default-rack"}; Priority mockPriority = Priority.newInstance(1); ArgumentCaptor requestCaptor = - ArgumentCaptor.forClass(CookieContainerRequest.class); + ArgumentCaptor.forClass(CookieContainerRequest.class); // allocate task scheduler.allocateTask(mockTask1, mockCapability, hosts, - racks, mockPriority, null, mockCookie1); + racks, mockPriority, null, mockCookie1); drainableAppCallback.drain(); verify(mockRMClient, times(1)). - addContainerRequest((CookieContainerRequest) any()); + addContainerRequest((CookieContainerRequest) any()); // returned from task requests before allocation happens assertFalse(scheduler.deallocateTask(mockTask1, true, null, null)); verify(mockApp, times(0)).containerBeingReleased(any()); verify(mockRMClient, times(1)). - removeContainerRequest((CookieContainerRequest) any()); + removeContainerRequest((CookieContainerRequest) any()); verify(mockRMClient, times(0)). - releaseAssignedContainer((ContainerId) any()); + releaseAssignedContainer((ContainerId) any()); // deallocating unknown task assertFalse(scheduler.deallocateTask(mockTask1, true, null, null)); verify(mockApp, times(0)).containerBeingReleased(any()); verify(mockRMClient, times(1)). - removeContainerRequest((CookieContainerRequest) any()); + removeContainerRequest((CookieContainerRequest) any()); verify(mockRMClient, times(0)). - releaseAssignedContainer((ContainerId) any()); + releaseAssignedContainer((ContainerId) any()); // allocate tasks Object mockTask2 = new MockTask("task2"); @@ -196,19 +196,19 @@ public void testTaskSchedulerNoReuse() throws Exception { racks, mockPriority, null, mockCookie1); drainableAppCallback.drain(); verify(mockRMClient, times(2)). - addContainerRequest(requestCaptor.capture()); + addContainerRequest(requestCaptor.capture()); CookieContainerRequest request1 = requestCaptor.getValue(); scheduler.allocateTask(mockTask2, mockCapability, hosts, racks, mockPriority, null, mockCookie2); drainableAppCallback.drain(); verify(mockRMClient, times(3)). - addContainerRequest(requestCaptor.capture()); + addContainerRequest(requestCaptor.capture()); CookieContainerRequest request2 = requestCaptor.getValue(); scheduler.allocateTask(mockTask3, mockCapability, hosts, racks, mockPriority, null, mockCookie3); drainableAppCallback.drain(); verify(mockRMClient, times(4)). - addContainerRequest(requestCaptor.capture()); + addContainerRequest(requestCaptor.capture()); CookieContainerRequest request3 = requestCaptor.getValue(); NodeId host1 = NodeId.newInstance("host1", 1); @@ -278,9 +278,9 @@ public void testTaskSchedulerNoReuse() throws Exception { // no other statuses returned verify(mockApp, times(3)).containerCompleted(any(), (ContainerStatus) any()); verify(mockRMClient, times(3)).releaseAssignedContainer((ContainerId) any()); - + // verify blacklisting - verify(mockRMClient, times(0)).addNodeToBlacklist((NodeId)any()); + verify(mockRMClient, times(0)).addNodeToBlacklist((NodeId) any()); String badHost = "host6"; NodeId badNodeId = NodeId.newInstance(badHost, 1); scheduler.blacklistNode(badNodeId); @@ -328,7 +328,7 @@ public void testTaskSchedulerNoReuse() throws Exception { float progress = 0.5f; when(mockApp.getProgress()).thenReturn(progress); Assert.assertEquals(progress, scheduler.getProgress(), 0); - + // check duplicate allocation request scheduler.allocateTask(mockTask1, mockCapability, hosts, racks, mockPriority, null, mockCookie1); @@ -390,12 +390,12 @@ public void testTaskSchedulerNoReuse() throws Exception { scheduler.shutdown(); drainableAppCallback.drain(); verify(mockRMClient). - unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, - SUCCEED_APP_MESSAGE, DEFAULT_APP_URL); + unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, + SUCCEED_APP_MESSAGE, DEFAULT_APP_URL); verify(mockRMClient).stop(); } - @Test(timeout=10000) + @Test(timeout = 10000) public void testTaskSchedulerInitiateStop() throws Exception { Configuration conf = new Configuration(); @@ -439,19 +439,19 @@ public void testTaskSchedulerInitiateStop() throws Exception { racks, mockPriority1, null, mockCookie1); drainableAppCallback.drain(); verify(mockRMClient, times(1)). - addContainerRequest(requestCaptor.capture()); + addContainerRequest(requestCaptor.capture()); CookieContainerRequest request1 = requestCaptor.getValue(); scheduler.allocateTask(mockTask2, mockCapability, hosts, racks, mockPriority2, null, mockCookie2); drainableAppCallback.drain(); verify(mockRMClient, times(2)). - addContainerRequest(requestCaptor.capture()); + addContainerRequest(requestCaptor.capture()); CookieContainerRequest request2 = requestCaptor.getValue(); scheduler.allocateTask(mockTask3, mockCapability, hosts, racks, mockPriority3, null, mockCookie3); drainableAppCallback.drain(); verify(mockRMClient, times(3)). - addContainerRequest(requestCaptor.capture()); + addContainerRequest(requestCaptor.capture()); CookieContainerRequest request3 = requestCaptor.getValue(); List containers = new ArrayList(); @@ -467,13 +467,13 @@ public void testTaskSchedulerInitiateStop() throws Exception { containers.add(mockContainer2); ArrayList hostContainers = - new ArrayList(); + new ArrayList(); hostContainers.add(request1); ArrayList rackContainers = - new ArrayList(); + new ArrayList(); rackContainers.add(request2); ArrayList anyContainers = - new ArrayList(); + new ArrayList(); anyContainers.add(request3); AtomicBoolean drainNotifier = new AtomicBoolean(false); @@ -499,8 +499,8 @@ public void testTaskSchedulerInitiateStop() throws Exception { verify(mockRMClient).removeContainerRequest(request3); } - @SuppressWarnings({ "unchecked" }) - @Test(timeout=10000) + @SuppressWarnings({"unchecked"}) + @Test(timeout = 10000) public void testTaskSchedulerWithReuse() throws Exception { TezAMRMClientAsync mockRMClient = spy( new AMRMClientAsyncForTest(new AMRMClientForTest(), 100)); @@ -518,7 +518,6 @@ public void testTaskSchedulerWithReuse() throws Exception { TaskSchedulerWithDrainableContext scheduler = new TaskSchedulerWithDrainableContext(drainableAppCallback, mockRMClient); - scheduler.initialize(); drainableAppCallback.drain(); scheduler.start(); @@ -545,19 +544,19 @@ public void testTaskSchedulerWithReuse() throws Exception { racks, mockPriority1, null, mockCookie1); drainableAppCallback.drain(); verify(mockRMClient, times(1)). - addContainerRequest(requestCaptor.capture()); + addContainerRequest(requestCaptor.capture()); CookieContainerRequest request1 = requestCaptor.getValue(); scheduler.allocateTask(mockTask2, mockCapability, hosts, racks, mockPriority2, null, mockCookie2); drainableAppCallback.drain(); verify(mockRMClient, times(2)). - addContainerRequest(requestCaptor.capture()); + addContainerRequest(requestCaptor.capture()); CookieContainerRequest request2 = requestCaptor.getValue(); scheduler.allocateTask(mockTask3, mockCapability, hosts, racks, mockPriority3, null, mockCookie3); drainableAppCallback.drain(); verify(mockRMClient, times(3)). - addContainerRequest(requestCaptor.capture()); + addContainerRequest(requestCaptor.capture()); CookieContainerRequest request3 = requestCaptor.getValue(); NodeId host1 = NodeId.newInstance("host1", 1); @@ -582,7 +581,7 @@ public void testTaskSchedulerWithReuse() throws Exception { AtomicBoolean drainNotifier = new AtomicBoolean(false); scheduler.delayedContainerManager.drainedDelayedContainersForTest = drainNotifier; - + scheduler.onContainersAllocated(containers); TestTaskSchedulerHelpers.waitForDelayedDrainNotify(drainNotifier); drainableAppCallback.drain(); @@ -635,7 +634,7 @@ public void testTaskSchedulerWithReuse() throws Exception { verify(mockRMClient, times(3)).releaseAssignedContainer((ContainerId) any()); // verify blacklisting - verify(mockRMClient, times(0)).addNodeToBlacklist((NodeId)any()); + verify(mockRMClient, times(0)).addNodeToBlacklist((NodeId) any()); String badHost = "host6"; NodeId badNodeId = NodeId.newInstance(badHost, 1); scheduler.blacklistNode(badNodeId); @@ -683,7 +682,7 @@ public void testTaskSchedulerWithReuse() throws Exception { scheduler.unblacklistNode(badNodeId); verify(mockRMClient, times(1)).removeNodeFromBlacklist(badNodeId); assertEquals(0, scheduler.blacklistedNodes.size()); - + // verify container level matching // add a dummy task to prevent release of allocated containers Object mockTask5 = new MockTask("task5"); @@ -705,7 +704,7 @@ public void testTaskSchedulerWithReuse() throws Exception { drainNotifier.set(false); scheduler.onContainersAllocated(containers); drainableAppCallback.drain(); - verify(mockRMClient, times(5)).releaseAssignedContainer((ContainerId) any()); + verify(mockRMClient, times(5)).releaseAssignedContainer((ContainerId) any()); Object mockTask6 = new MockTask("task6"); Object mockCookie6 = new Object(); // allocate request with container affinity @@ -726,7 +725,6 @@ public void testTaskSchedulerWithReuse() throws Exception { verify(mockRMClient).releaseAssignedContainer(mockCId7); verify(mockRMClient).releaseAssignedContainer(mockCId8); verify(mockRMClient, times(7)).releaseAssignedContainer((ContainerId) any()); - float progress = 0.5f; when(mockApp.getProgress()).thenReturn(progress); @@ -737,13 +735,12 @@ public void testTaskSchedulerWithReuse() throws Exception { drainableAppCallback.drain(); verify(mockApp).nodesUpdated(mockUpdatedNodes); - ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(String.class); Exception mockException = new IOException("mockexception"); scheduler.onError(mockException); drainableAppCallback.drain(); verify(mockApp).reportError(eq(YarnTaskSchedulerServiceError.RESOURCEMANAGER_ERROR), argumentCaptor.capture(), - any()); + any()); assertTrue(argumentCaptor.getValue().contains("mockexception")); scheduler.onShutdownRequest(); @@ -756,18 +753,18 @@ public void testTaskSchedulerWithReuse() throws Exception { scheduler.shutdown(); drainableAppCallback.drain(); verify(mockRMClient). - unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, - SUCCEED_APP_MESSAGE, DEFAULT_APP_URL); + unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, + SUCCEED_APP_MESSAGE, DEFAULT_APP_URL); verify(mockRMClient).stop(); } - - @Test (timeout=5000) + + @Test(timeout = 5000) public void testTaskSchedulerDetermineMinHeldContainers() throws Exception { TezAMRMClientAsync mockRMClient = spy( new AMRMClientAsyncForTest(new AMRMClientForTest(), 100)); TaskSchedulerContext mockApp = setupMockTaskSchedulerContext(DEFAULT_APP_HOST, DEFAULT_APP_PORT, DEFAULT_APP_URL, - true, new Configuration()); + true, new Configuration()); final TaskSchedulerContextDrainable drainableAppCallback = createDrainableContext(mockApp); TaskSchedulerWithDrainableContext scheduler = @@ -775,7 +772,7 @@ public void testTaskSchedulerDetermineMinHeldContainers() throws Exception { scheduler.initialize(); scheduler.start(); - + String rack1 = "r1"; String rack2 = "r2"; String rack3 = "r3"; @@ -838,22 +835,22 @@ public void testTaskSchedulerDetermineMinHeldContainers() throws Exception { scheduler.heldContainers.put(mockCId5, hc5); scheduler.heldContainers.put(mockCId6, hc6); scheduler.heldContainers.put(mockCId7, hc7); - + // test empty case scheduler.sessionNumMinHeldContainers = 0; scheduler.determineMinHeldContainers(); Assert.assertEquals(0, scheduler.sessionMinHeldContainers.size()); - + // test min >= held scheduler.sessionNumMinHeldContainers = 7; scheduler.determineMinHeldContainers(); Assert.assertEquals(7, scheduler.sessionMinHeldContainers.size()); - + // test min < held scheduler.sessionNumMinHeldContainers = 5; scheduler.determineMinHeldContainers(); Assert.assertEquals(5, scheduler.sessionMinHeldContainers.size()); - + Set heldContainers = Sets.newHashSet(); for (ContainerId cId : scheduler.sessionMinHeldContainers) { heldContainers.add(scheduler.heldContainers.get(cId)); @@ -875,7 +872,7 @@ public void testTaskSchedulerDetermineMinHeldContainers() throws Exception { Assert.assertEquals(3, racks.size()); Assert.assertTrue(racks.contains(rack1) && racks.contains(rack2) && racks.contains(rack3)); - + long currTime = System.currentTimeMillis(); heldContainers.clear(); heldContainers.addAll(scheduler.heldContainers.values()); @@ -886,7 +883,7 @@ public void testTaskSchedulerDetermineMinHeldContainers() throws Exception { Thread.sleep(1000); drainableAppCallback.drain(); // only the 2 container not in min-held containers are released - verify(mockRMClient, times(2)).releaseAssignedContainer((ContainerId)any()); + verify(mockRMClient, times(2)).releaseAssignedContainer((ContainerId) any()); Assert.assertEquals(5, scheduler.heldContainers.size()); AppFinalStatus finalStatus = @@ -895,15 +892,15 @@ public void testTaskSchedulerDetermineMinHeldContainers() throws Exception { scheduler.shutdown(); } - @Test (timeout=3000) + @Test(timeout = 3000) public void testTaskSchedulerHeldContainersReleaseAfterExpired() throws Exception { final TezAMRMClientAsync mockRMClient = spy( - new AMRMClientAsyncForTest(new AMRMClientForTest(), 100)); + new AMRMClientAsyncForTest(new AMRMClientForTest(), 100)); final TaskSchedulerContext mockApp = setupMockTaskSchedulerContext(DEFAULT_APP_HOST, DEFAULT_APP_PORT, - DEFAULT_APP_URL, true, new Configuration()); + DEFAULT_APP_URL, true, new Configuration()); final TaskSchedulerContextDrainable drainableAppCallback = createDrainableContext(mockApp); final TaskSchedulerWithDrainableContext scheduler = - new TaskSchedulerWithDrainableContext(drainableAppCallback, mockRMClient); + new TaskSchedulerWithDrainableContext(drainableAppCallback, mockRMClient); scheduler.initialize(); scheduler.start(); @@ -924,16 +921,16 @@ public void testTaskSchedulerHeldContainersReleaseAfterExpired() throws Exceptio // sleep and wait for mainLoop() check-in to release this expired held container. Thread.sleep(1000); - verify(mockRMClient, times(1)).releaseAssignedContainer((ContainerId)any()); + verify(mockRMClient, times(1)).releaseAssignedContainer((ContainerId) any()); Assert.assertEquals(0, scheduler.heldContainers.size()); AppFinalStatus finalStatus = - new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, SUCCEED_APP_MESSAGE, DEFAULT_APP_URL); + new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, SUCCEED_APP_MESSAGE, DEFAULT_APP_URL); when(mockApp.getFinalAppStatus()).thenReturn(finalStatus); scheduler.shutdown(); } - - @Test(timeout=5000) + + @Test(timeout = 5000) public void testTaskSchedulerRandomReuseExpireTime() throws Exception { TezAMRMClientAsync mockRMClient = spy( new AMRMClientAsyncForTest(new AMRMClientForTest(), 100)); @@ -953,7 +950,6 @@ public void testTaskSchedulerRandomReuseExpireTime() throws Exception { final TaskSchedulerContextDrainable drainableAppCallback1 = createDrainableContext(mockApp1); final TaskSchedulerContextDrainable drainableAppCallback2 = createDrainableContext(mockApp2); - TaskSchedulerWithDrainableContext scheduler1 = new TaskSchedulerWithDrainableContext(drainableAppCallback1, mockRMClient); TaskSchedulerWithDrainableContext scheduler2 = @@ -964,15 +960,15 @@ public void testTaskSchedulerRandomReuseExpireTime() throws Exception { scheduler1.start(); scheduler2.start(); - + // when min == max the expire time is always min - for (int i=0; i<10; ++i) { + for (int i = 0; i < 10; ++i) { Assert.assertEquals(minTime, scheduler1.getHeldContainerExpireTime(0)); } - + long lastExpireTime = 0; // when min < max the expire time is random in between min and max - for (int i=0; i<10; ++i) { + for (int i = 0; i < 10; ++i) { long currExpireTime = scheduler2.getHeldContainerExpireTime(0); Assert.assertTrue( "min: " + minTime + " curr: " + currExpireTime + " max: " + maxTime, @@ -989,18 +985,18 @@ public void testTaskSchedulerRandomReuseExpireTime() throws Exception { scheduler2.shutdown(); } - @SuppressWarnings({ "unchecked", "rawtypes" }) - @Test (timeout=5000) + @SuppressWarnings({"unchecked", "rawtypes"}) + @Test(timeout = 5000) public void testTaskSchedulerPreemption() throws Exception { TezAMRMClientAsync mockRMClient = - mock(TezAMRMClientAsync.class); + mock(TezAMRMClientAsync.class); Configuration conf = new Configuration(); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, false); conf.setInt(TezConfiguration.TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS, 3); TaskSchedulerContext mockApp = setupMockTaskSchedulerContext(DEFAULT_APP_HOST, DEFAULT_APP_PORT, DEFAULT_APP_URL, - false, null, null, new PreemptionMatcher(), conf); + false, null, null, new PreemptionMatcher(), conf); final TaskSchedulerContextDrainable drainableAppCallback = createDrainableContext(mockApp); final TaskSchedulerWithDrainableContext scheduler = @@ -1009,7 +1005,7 @@ public void testTaskSchedulerPreemption() throws Exception { scheduler.initialize(); RegisterApplicationMasterResponse mockRegResponse = - mock(RegisterApplicationMasterResponse.class); + mock(RegisterApplicationMasterResponse.class); when( mockRMClient.registerApplicationMaster(anyString(), anyInt(), anyString())).thenReturn(mockRegResponse); @@ -1022,7 +1018,7 @@ public void testTaskSchedulerPreemption() throws Exception { scheduler.getProgress(); drainableAppCallback.drain(); Assert.assertEquals(totalResource, scheduler.getTotalResources()); - verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId)any()); + verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any()); // allocate task Object mockTask1 = new MockTask("task1"); @@ -1045,33 +1041,32 @@ public void testTaskSchedulerPreemption() throws Exception { final ArrayList anyContainers = new ArrayList(); - Resource taskAsk = Resource.newInstance(1024, 1); scheduler.allocateTask(mockTask1, taskAsk, null, - null, pri2, null, null); + null, pri2, null, null); drainableAppCallback.drain(); verify(mockRMClient, times(1)). addContainerRequest(requestCaptor.capture()); anyContainers.add(requestCaptor.getValue()); scheduler.allocateTask(mockTask3, taskAsk, null, - null, pri6, obj3, null); + null, pri6, obj3, null); drainableAppCallback.drain(); verify(mockRMClient, times(2)). - addContainerRequest(requestCaptor.capture()); + addContainerRequest(requestCaptor.capture()); anyContainers.add(requestCaptor.getValue()); // later one in the allocation gets killed between the two task3's scheduler.allocateTask(mockTask3KillA, taskAsk, null, - null, pri6, obj3, null); + null, pri6, obj3, null); drainableAppCallback.drain(); verify(mockRMClient, times(3)). - addContainerRequest(requestCaptor.capture()); + addContainerRequest(requestCaptor.capture()); anyContainers.add(requestCaptor.getValue()); // later one in the allocation gets killed between the two task3's scheduler.allocateTask(mockTask3KillB, taskAsk, null, - null, pri6, obj3, null); + null, pri6, obj3, null); drainableAppCallback.drain(); verify(mockRMClient, times(4)). - addContainerRequest(requestCaptor.capture()); + addContainerRequest(requestCaptor.capture()); anyContainers.add(requestCaptor.getValue()); Resource freeResource = Resource.newInstance(500, 0); @@ -1079,7 +1074,7 @@ public void testTaskSchedulerPreemption() throws Exception { scheduler.getProgress(); drainableAppCallback.drain(); Assert.assertEquals(totalResource, scheduler.getTotalResources()); - verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId)any()); + verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any()); final List> anyList = new LinkedList>(); @@ -1111,7 +1106,6 @@ public List> answer( InvocationOnMock invocation) throws Throwable { return emptyList; } - }); // RackResolver by default puts hosts in default-rack when( @@ -1123,34 +1117,34 @@ public List> answer( InvocationOnMock invocation) throws Throwable { return emptyList; } - }); when( mockRMClient.getMatchingRequests((Priority) any(), eq(ResourceRequest.ANY), (Resource) any())).thenAnswer( new Answer>>() { int calls = 0; + @Override public List> answer( InvocationOnMock invocation) throws Throwable { - if(calls > 0) { + if (calls > 0) { anyContainers.remove(0); } calls++; return anyList; } - }); - + doAnswer(new Answer() { public Object answer(InvocationOnMock invocation) { - Object[] args = invocation.getArguments(); - ContainerId cId = (ContainerId) args[0]; - scheduler.deallocateContainer(cId); - return null; - }}) - .when(mockApp).preemptContainer((ContainerId)any()); - + Object[] args = invocation.getArguments(); + ContainerId cId = (ContainerId) args[0]; + scheduler.deallocateContainer(cId); + return null; + } + }) + .when(mockApp).preemptContainer((ContainerId) any()); + scheduler.onContainersAllocated(containers); drainableAppCallback.drain(); Assert.assertEquals(4, scheduler.taskAllocations.size()); @@ -1169,29 +1163,29 @@ public Object answer(InvocationOnMock invocation) { // no preemption scheduler.getProgress(); drainableAppCallback.drain(); - verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId)any()); + verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any()); // no need for task preemption until now - so they should match Assert.assertEquals(scheduler.numHeartbeats, scheduler.heartbeatAtLastPreemption); // add a pending request that cannot be allocated until resources free up Object mockTask3WaitCookie = new Object(); scheduler.allocateTask(mockTask3Wait, taskAsk, null, - null, pri6, obj3, mockTask3WaitCookie); + null, pri6, obj3, mockTask3WaitCookie); // add a pri 8 request for the pri 8 container that will not be matched Object mockTaskPri8Cookie = new Object(); scheduler.allocateTask(mockTaskPri8, taskAsk, null, - null, pri8, obj3, mockTaskPri8Cookie); + null, pri8, obj3, mockTaskPri8Cookie); // no preemption - same pri scheduler.getProgress(); drainableAppCallback.drain(); verify(mockRMClient, times(6)).addContainerRequest(requestCaptor.capture()); - verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId)any()); + verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any()); ContainerId mockCId5 = ContainerId.newContainerId(attemptId, 5); Container mockContainer5 = Container.newInstance(mockCId5, host1, null, taskAsk, pri8, null); containers.clear(); containers.add(mockContainer5); - + // new lower pri container added that wont be matched and eventually preempted // Fudge new container being present in delayed allocation list due to race HeldContainer heldContainer = new HeldContainer(mockContainer5, -1, -1, null, @@ -1200,7 +1194,7 @@ public Object answer(InvocationOnMock invocation) { // no preemption - container assignment attempts < 3 scheduler.getProgress(); drainableAppCallback.drain(); - verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId)any()); + verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any()); // no need for task preemption until now - so they should match Assert.assertEquals(scheduler.numHeartbeats, scheduler.heartbeatAtLastPreemption); @@ -1208,13 +1202,13 @@ public Object answer(InvocationOnMock invocation) { // no preemption - container assignment attempts < 3 scheduler.getProgress(); drainableAppCallback.drain(); - verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId)any()); + verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any()); heldContainer.incrementAssignmentAttempts(); heldContainer.incrementAssignmentAttempts(); // preemption - container released and resource asked again scheduler.getProgress(); drainableAppCallback.drain(); - verify(mockRMClient, times(1)).releaseAssignedContainer((ContainerId)any()); + verify(mockRMClient, times(1)).releaseAssignedContainer((ContainerId) any()); verify(mockRMClient, times(1)).releaseAssignedContainer(mockCId5); // internally re-request pri8 task request because we release pri8 new container verify(mockRMClient, times(7)).addContainerRequest(requestCaptor.capture()); @@ -1222,7 +1216,7 @@ public Object answer(InvocationOnMock invocation) { Assert.assertEquals(pri8, reAdded.getPriority()); Assert.assertEquals(taskAsk, reAdded.getCapability()); Assert.assertEquals(mockTaskPri8Cookie, reAdded.getCookie().getAppCookie()); - + // remove fudging. scheduler.delayedContainerManager.delayedContainers.clear(); @@ -1230,18 +1224,18 @@ public Object answer(InvocationOnMock invocation) { Assert.assertEquals(scheduler.numHeartbeats, scheduler.heartbeatAtLastPreemption); scheduler.allocateTask(mockTask3Retry, taskAsk, null, - null, pri5, obj3, null); + null, pri5, obj3, null); // no preemption - higher pri. exact match scheduler.getProgress(); // no need for task preemption until now - so they should match drainableAppCallback.drain(); // no need for task preemption until now - so they should match Assert.assertEquals(scheduler.numHeartbeats, scheduler.heartbeatAtLastPreemption); - verify(mockRMClient, times(1)).releaseAssignedContainer((ContainerId)any()); + verify(mockRMClient, times(1)).releaseAssignedContainer((ContainerId) any()); - for (int i=0; i<11; ++i) { + for (int i = 0; i < 11; ++i) { scheduler.allocateTask(mockTask2, taskAsk, null, - null, pri4, null, null); + null, pri4, null, null); } drainableAppCallback.drain(); @@ -1254,10 +1248,10 @@ public Object answer(InvocationOnMock invocation) { drainableAppCallback.drain(); scheduler.getProgress(); // second heartbeat drainableAppCallback.drain(); - verify(mockRMClient, times(1)).releaseAssignedContainer((ContainerId)any()); + verify(mockRMClient, times(1)).releaseAssignedContainer((ContainerId) any()); scheduler.getProgress(); // third heartbeat drainableAppCallback.drain(); - verify(mockRMClient, times(2)).releaseAssignedContainer((ContainerId)any()); + verify(mockRMClient, times(2)).releaseAssignedContainer((ContainerId) any()); verify(mockRMClient, times(1)).releaseAssignedContainer(mockCId4); Assert.assertEquals(scheduler.numHeartbeats, scheduler.heartbeatAtLastPreemption); // there are pending preemptions. @@ -1267,7 +1261,7 @@ public Object answer(InvocationOnMock invocation) { scheduler.getProgress(); // third heartbeat drainableAppCallback.drain(); // Next oldest mockTaskPri3KillA gets preempted to clear 10% of outstanding running preemptable tasks - verify(mockRMClient, times(3)).releaseAssignedContainer((ContainerId)any()); + verify(mockRMClient, times(3)).releaseAssignedContainer((ContainerId) any()); verify(mockRMClient, times(1)).releaseAssignedContainer(mockCId3); AppFinalStatus finalStatus = @@ -1276,21 +1270,21 @@ public Object answer(InvocationOnMock invocation) { scheduler.shutdown(); drainableAppCallback.drain(); } - - @Test (timeout=5000) + + @Test(timeout = 5000) public void testTaskSchedulerPreemption2() throws Exception { TezAMRMClientAsync mockRMClient = spy( new AMRMClientAsyncForTest(new AMRMClientForTest(), 100)); int waitTime = 1000; - + Configuration conf = new Configuration(); conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, false); conf.setInt(TezConfiguration.TEZ_AM_PREEMPTION_HEARTBEATS_BETWEEN_PREEMPTIONS, 2); conf.setInt(TezConfiguration.TEZ_AM_PREEMPTION_MAX_WAIT_TIME_MS, waitTime); TaskSchedulerContext mockApp = setupMockTaskSchedulerContext(DEFAULT_APP_HOST, DEFAULT_APP_PORT, DEFAULT_APP_URL, - false, null, null, new PreemptionMatcher(), conf); + false, null, null, new PreemptionMatcher(), conf); final TaskSchedulerContextDrainable drainableAppCallback = createDrainableContext(mockApp); final TaskSchedulerWithDrainableContext scheduler = @@ -1304,7 +1298,7 @@ public void testTaskSchedulerPreemption2() throws Exception { drainableAppCallback.drain(); Resource totalResource = mockRMClient.getAvailableResources(); Assert.assertEquals(totalResource, scheduler.getTotalResources()); - verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId)any()); + verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any()); // allocate task Object mockTask1 = new MockTask("task1"); @@ -1320,10 +1314,9 @@ public void testTaskSchedulerPreemption2() throws Exception { final ArrayList anyContainers = new ArrayList(); - Resource taskAsk = Resource.newInstance(1024, 1); scheduler.allocateTask(mockTask1, taskAsk, null, - null, pri4, null, null); + null, pri4, null, null); drainableAppCallback.drain(); verify(mockRMClient, times(1)). addContainerRequest(requestCaptor.capture()); @@ -1332,7 +1325,7 @@ public void testTaskSchedulerPreemption2() throws Exception { scheduler.getProgress(); drainableAppCallback.drain(); Assert.assertEquals(totalResource, scheduler.getTotalResources()); - verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId)any()); + verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any()); NodeId host1 = NodeId.newInstance("host1", 1); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1); @@ -1340,16 +1333,17 @@ public void testTaskSchedulerPreemption2() throws Exception { Container mockContainer1 = Container.newInstance(mockCId1, host1, null, taskAsk, pri4, null); List containers = new ArrayList(); containers.add(mockContainer1); - + doAnswer(new Answer() { public Object answer(InvocationOnMock invocation) { - Object[] args = invocation.getArguments(); - ContainerId cId = (ContainerId) args[0]; - scheduler.deallocateContainer(cId); - return null; - }}) - .when(mockApp).preemptContainer((ContainerId)any()); - + Object[] args = invocation.getArguments(); + ContainerId cId = (ContainerId) args[0]; + scheduler.deallocateContainer(cId); + return null; + } + }) + .when(mockApp).preemptContainer((ContainerId) any()); + scheduler.onContainersAllocated(containers); drainableAppCallback.drain(); Assert.assertEquals(1, scheduler.taskAllocations.size()); @@ -1359,17 +1353,17 @@ public Object answer(InvocationOnMock invocation) { // no preemption scheduler.getProgress(); drainableAppCallback.drain(); - verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId)any()); + verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any()); // no need for task preemption until now - so they should match Assert.assertEquals(scheduler.numHeartbeats, scheduler.heartbeatAtLastPreemption); // add a pending request that cannot be allocated until resources free up Object mockTask2Cookie = new Object(); scheduler.allocateTask(mockTask2, taskAsk, null, - null, pri2, obj3, mockTask2Cookie); + null, pri2, obj3, mockTask2Cookie); Object mockTask3Cookie = new Object(); scheduler.allocateTask(mockTask3, taskAsk, null, - null, pri6, obj3, mockTask3Cookie); + null, pri6, obj3, mockTask3Cookie); // nothing waiting till now Assert.assertNull(scheduler.highestWaitingRequestPriority); Assert.assertEquals(0, scheduler.highestWaitingRequestWaitStartTime); @@ -1377,18 +1371,18 @@ public Object answer(InvocationOnMock invocation) { long currTime = System.currentTimeMillis(); scheduler.getProgress(); drainableAppCallback.drain(); - verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId)any()); + verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any()); // enough free resources. preemption not triggered Assert.assertEquals(pri2, scheduler.highestWaitingRequestPriority); Assert.assertTrue(scheduler.highestWaitingRequestWaitStartTime >= currTime); Assert.assertEquals(scheduler.numHeartbeats, scheduler.heartbeatAtLastPreemption); - + Thread.sleep(waitTime + 10); long oldStartWaitTime = scheduler.highestWaitingRequestWaitStartTime; - + scheduler.getProgress(); drainableAppCallback.drain(); - verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId)any()); + verify(mockRMClient, times(0)).releaseAssignedContainer((ContainerId) any()); // enough free resources. deadline crossed. preemption triggered Assert.assertEquals(pri2, scheduler.highestWaitingRequestPriority); Assert.assertEquals(oldStartWaitTime, scheduler.highestWaitingRequestWaitStartTime); @@ -1396,7 +1390,7 @@ public Object answer(InvocationOnMock invocation) { scheduler.getProgress(); drainableAppCallback.drain(); - verify(mockRMClient, times(1)).releaseAssignedContainer((ContainerId)any()); + verify(mockRMClient, times(1)).releaseAssignedContainer((ContainerId) any()); verify(mockRMClient, times(1)).releaseAssignedContainer(mockCId1); Assert.assertEquals(scheduler.numHeartbeats, scheduler.heartbeatAtLastPreemption); // maintains existing waiting values @@ -1405,7 +1399,7 @@ public Object answer(InvocationOnMock invocation) { // remove high pri request to test waiting pri change scheduler.deallocateTask(mockTask2, false, null, null); - + scheduler.getProgress(); // waiting value changes Assert.assertEquals(pri6, scheduler.highestWaitingRequestPriority); @@ -1447,15 +1441,15 @@ public void testLocalityMatching() throws Exception { taskScheduler.initialize(); taskScheduler.start(); - + Resource resource = Resource.newInstance(1024, 1); Priority priority = Priority.newInstance(1); - String hostsTask1[] = { "host1" }; - String hostsTask2[] = { "non-allocated-host" }; + String hostsTask1[] = {"host1"}; + String hostsTask2[] = {"non-allocated-host"}; - String defaultRack[] = { "/default-rack" }; - String otherRack[] = { "/other-rack" }; + String defaultRack[] = {"/default-rack"}; + String otherRack[] = {"/other-rack"}; Object mockTask1 = new MockTask("task1"); CookieContainerRequest mockCookie1 = mock(CookieContainerRequest.class, @@ -1525,8 +1519,8 @@ public void testLocalityMatching() throws Exception { when(appClient.getFinalAppStatus()).thenReturn(finalStatus); taskScheduler.shutdown(); } - - @Test (timeout=5000) + + @Test(timeout = 5000) public void testScaleDownPercentage() { Assert.assertEquals(100, YarnTaskSchedulerService.scaleDownByPreemptionPercentage(100, 100)); Assert.assertEquals(70, YarnTaskSchedulerService.scaleDownByPreemptionPercentage(100, 70)); @@ -1577,14 +1571,14 @@ public void testContainerExpired() throws Exception { racks, mockPriority2, null, mockCookie2); drainableAppCallback.drain(); verify(mockRMClient, times(1)). - addContainerRequest(requestCaptor.capture()); + addContainerRequest(requestCaptor.capture()); CookieContainerRequest request2 = requestCaptor.getValue(); scheduler.allocateTask(mockTask1, mockCapability, hosts, racks, mockPriority1, null, mockCookie1); drainableAppCallback.drain(); verify(mockRMClient, times(2)). - addContainerRequest(requestCaptor.capture()); + addContainerRequest(requestCaptor.capture()); List containers = new ArrayList(); // sending only lower priority container to make sure its not matched @@ -1620,7 +1614,7 @@ public void testContainerExpired() throws Exception { } private Container createContainer(int id, String host, Resource resource, - Priority priority) { + Priority priority) { ContainerId containerID = ContainerId.newInstance( ApplicationAttemptId.newInstance(ApplicationId.newInstance(1, 1), 1), id); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java index b7acc6876c..c89146ac4c 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerHelpers.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -95,7 +95,6 @@ protected void serviceStop() { } } - // Mocking AMRMClientAsyncImpl to make use of getMatchingRequest static class AMRMClientAsyncForTest extends TezAMRMClientAsync { @@ -123,7 +122,7 @@ public RegisterApplicationMasterResponse registerApplicationMaster( @Override public void unregisterApplicationMaster(FinalApplicationStatus appStatus, - String appMessage, String appTrackingUrl) { + String appMessage, String appTrackingUrl) { } @Override @@ -138,7 +137,7 @@ RegisterApplicationMasterResponse getRegistrationResponse() { return mockRegResponse; } } - + // Overrides start / stop. Will be controlled without the extra event handling thread. static class TaskSchedulerManagerForTest extends TaskSchedulerManager { @@ -228,7 +227,6 @@ public Event verifyInvocation(Class eventClass) { static class TaskSchedulerWithDrainableContext extends YarnTaskSchedulerService { - public TaskSchedulerWithDrainableContext( TaskSchedulerContextDrainable appClient, TezAMRMClientAsync client) { @@ -237,7 +235,7 @@ public TaskSchedulerWithDrainableContext( } public TaskSchedulerContextDrainable getDrainableAppCallback() { - return (TaskSchedulerContextDrainable)getContext(); + return (TaskSchedulerContextDrainable) getContext(); } } @@ -248,7 +246,7 @@ static class TaskSchedulerContextDrainable implements TaskSchedulerContext { private TaskSchedulerContext real; private CountingExecutorService countingExecutorService; final AtomicInteger count = new AtomicInteger(0); - + public TaskSchedulerContextDrainable(TaskSchedulerContextImplWrapper real) { countingExecutorService = (CountingExecutorService) real.getExecutorService(); this.real = real; @@ -263,7 +261,7 @@ public void taskAllocated(Object task, Object appCookie, Container container) { @Override public void containerCompleted(Object taskLastAllocated, - ContainerStatus containerStatus) { + ContainerStatus containerStatus) { invocations++; real.containerCompleted(taskLastAllocated, containerStatus); } @@ -288,7 +286,7 @@ public void appShutdownRequested() { @Override public void setApplicationRegistrationData(Resource maxContainerCapability, - Map appAcls, ByteBuffer key, String queueName) { + Map appAcls, ByteBuffer key, String queueName) { invocations++; real.setApplicationRegistrationData(maxContainerCapability, appAcls, key, queueName); } @@ -404,7 +402,7 @@ public boolean isExactMatch(Object cs1, Object cs2) { @Override public Map getAdditionalResources(Map lr1, - Map lr2) { + Map lr2) { return Maps.newHashMap(); } @@ -413,7 +411,7 @@ public Object union(Object cs1, Object cs2) { return cs1; } } - + static class PreemptionMatcher implements ContainerSignatureMatcher { @Override public boolean isSuperSet(Object cs1, Object cs2) { @@ -432,7 +430,7 @@ public boolean isExactMatch(Object cs1, Object cs2) { @Override public Map getAdditionalResources(Map lr1, - Map lr2) { + Map lr2) { return Maps.newHashMap(); } @@ -441,7 +439,6 @@ public Object union(Object cs1, Object cs2) { return cs1; } } - static void waitForDelayedDrainNotify(AtomicBoolean drainNotifier) throws InterruptedException { @@ -520,7 +517,7 @@ public List> invokeAll(Collection> tasks) @Override public List> invokeAll(Collection> tasks, long timeout, - TimeUnit unit) throws InterruptedException { + TimeUnit unit) throws InterruptedException { throw new UnsupportedOperationException("Not expected to be used"); } @@ -586,5 +583,4 @@ static TaskSchedulerContext setupMockTaskSchedulerContext(String appHost, int ap return mockContext; } - } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java index 095bd4e287..6ed5211b96 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskSchedulerManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -115,19 +115,20 @@ @SuppressWarnings("rawtypes") public class TestTaskSchedulerManager { - - class TestEventHandler implements EventHandler{ + + class TestEventHandler implements EventHandler { List events = Lists.newLinkedList(); + @Override public void handle(Event event) { events.add(event); } } - + class MockTaskSchedulerManager extends TaskSchedulerManager { final AtomicBoolean notify = new AtomicBoolean(false); - + public MockTaskSchedulerManager(AppContext appContext, DAGClientServer clientService, EventHandler eventHandler, ContainerSignatureMatcher containerSignatureMatcher, @@ -144,7 +145,7 @@ protected void instantiateSchedulers(String host, int port, String trackingUrl, taskSchedulerServiceWrappers[0] = new ServicePluginLifecycleAbstractService<>(taskSchedulers[0].getTaskScheduler()); } - + @Override protected void notifyForTest() { synchronized (notify) { @@ -152,7 +153,6 @@ protected void notifyForTest() { notify.notifyAll(); } } - } AppContext mockAppContext; @@ -292,7 +292,7 @@ public void testTAUnsuccessfulAfterContainerCleanup() throws Exception { assertEquals(mockCId, stopEvent.getContainerId()); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testTaskBasedAffinity() throws Exception { Configuration conf = new Configuration(false); schedulerHandler.init(conf); @@ -322,21 +322,21 @@ public void testTaskBasedAffinity() throws Exception { schedulerHandler.notify.wait(); } } - + // verify mockTaskAttempt affinitized to expected affCId verify(mockTaskScheduler, times(1)).allocateTask(mockTaskAttempt, resource, affCId, Priority.newInstance(3), null, event); - + schedulerHandler.stop(); schedulerHandler.close(); } - - @Test (timeout = 5000) + + @Test(timeout = 5000) public void testContainerPreempted() throws IOException { Configuration conf = new Configuration(false); schedulerHandler.init(conf); schedulerHandler.start(); - + String diagnostics = "Container preempted by RM."; TaskAttemptImpl mockTask = mock(TaskAttemptImpl.class); ContainerStatus mockStatus = mock(ContainerStatus.class); @@ -364,7 +364,7 @@ public void testContainerPreempted() throws IOException { schedulerHandler.close(); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testContainerInternalPreempted() throws IOException, ServicePluginException { Configuration conf = new Configuration(false); schedulerHandler.init(conf); @@ -424,13 +424,13 @@ public void testContainerInternalPreemptedAfterContainerCleanup() throws IOExcep schedulerHandler.stop(); schedulerHandler.close(); } - - @Test (timeout = 5000) + + @Test(timeout = 5000) public void testContainerDiskFailed() throws IOException { Configuration conf = new Configuration(false); schedulerHandler.init(conf); schedulerHandler.start(); - + String diagnostics = "NM disk failed."; TaskAttemptImpl mockTask = mock(TaskAttemptImpl.class); ContainerStatus mockStatus = mock(ContainerStatus.class); @@ -458,7 +458,7 @@ public void testContainerDiskFailed() throws IOException { schedulerHandler.close(); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testContainerExceededPMem() throws IOException { Configuration conf = new Configuration(false); schedulerHandler.init(conf); @@ -493,7 +493,7 @@ public void testContainerExceededPMem() throws IOException { schedulerHandler.close(); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testHistoryUrlConf() throws Exception { Configuration conf = schedulerHandler.appContext.getAMConf(); final ApplicationId mockApplicationId = mock(ApplicationId.class); @@ -526,10 +526,9 @@ public void testHistoryUrlConf() throws Exception { conf.set(TezConfiguration.TEZ_HISTORY_URL_BASE, "http://localhost/ui/tez"); assertEquals("http://localhost/ui/tez?viewPath=tez-app/TEST_APP_ID", schedulerHandler.getHistoryUrl()); - } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testHistoryUrlWithoutScheme() throws Exception { Configuration conf = schedulerHandler.appContext.getAMConf(); final ApplicationId mockApplicationId = mock(ApplicationId.class); @@ -686,7 +685,7 @@ public void testShutdownBeforeStartTaskScheduler() { TaskSchedulerManager taskSchedulerManager = new TaskSchedulerManager(appContext, null, null, - null, null, list, false,null); + null, null, list, false, null); assertFalse("Should not return true unless actually unregistered successfully", taskSchedulerManager.hasUnregistered()); } @@ -745,7 +744,6 @@ TaskSchedulerContext wrapTaskSchedulerContext(TaskSchedulerContext rawContext) { .contains(ServicePluginErrorDefaults.SERVICE_UNAVAILABLE.name())); assertTrue(killEvent.getDiagnosticInfo().contains(expIdentifier)); - reset(eventHandler); taskSchedulerManager.getAvailableResources(0); argumentCaptor = ArgumentCaptor.forClass(Event.class); @@ -761,7 +759,6 @@ TaskSchedulerContext wrapTaskSchedulerContext(TaskSchedulerContext rawContext) { assertTrue( event.getDiagnosticInfo().contains(ServicePluginErrorDefaults.INCONSISTENT_STATE.name())); assertTrue(event.getDiagnosticInfo().contains(expIdentifier)); - } finally { taskSchedulerManager.stop(); } @@ -820,7 +817,6 @@ protected void instantiateSchedulers(String host, int port, String trackingUrl, assertTrue(event.getDiagnosticInfo().contains("Task Allocation")); assertTrue(event.getDiagnosticInfo().contains(expectedId)); - taskSchedulerManager.dagCompleted(); argumentCaptor = ArgumentCaptor.forClass(Event.class); verify(eventHandler, times(2)).handle(argumentCaptor.capture()); @@ -833,7 +829,6 @@ protected void instantiateSchedulers(String host, int port, String trackingUrl, assertTrue(event.getError().getMessage().contains("TestException_" + "dagComplete")); assertTrue(event.getDiagnosticInfo().contains("Dag Completion")); assertTrue(event.getDiagnosticInfo().contains(expectedId)); - } finally { taskSchedulerManager.stop(); } @@ -919,7 +914,7 @@ TaskScheduler createUberTaskScheduler(TaskSchedulerContext taskSchedulerContext, @Override TaskScheduler createCustomTaskScheduler(TaskSchedulerContext taskSchedulerContext, NamedEntityDescriptor taskSchedulerDescriptor, int schedulerId) - throws TezException { + throws TezException { taskSchedulerContexts.add(taskSchedulerContext); TaskScheduler taskScheduler = spy(super.createCustomTaskScheduler(taskSchedulerContext, taskSchedulerDescriptor, schedulerId)); testTaskSchedulers.add(taskScheduler); @@ -1031,6 +1026,7 @@ public boolean hasUnregistered() { private static final String DAG_NAME = "dagName"; private static final int DAG_INDEX = 1; + public static class TaskSchedulerForFailureTest extends TaskScheduler { public TaskSchedulerForFailureTest(TaskSchedulerContext taskSchedulerContext) { diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTezAMRMClient.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTezAMRMClient.java index 9bb3aff81e..0d2234cadf 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTezAMRMClient.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTezAMRMClient.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -50,7 +50,7 @@ public static void beforeClass() { @Before public void setup() { amrmClient = new TezAMRMClientAsync(new AMRMClientImpl(), - 1000, mock(AMRMClientAsync.CallbackHandler.class)); + 1000, mock(AMRMClientAsync.CallbackHandler.class)); RackResolver.init(new Configuration()); } @@ -60,29 +60,29 @@ public void teardown() { } @SuppressWarnings("unchecked") - @Test(timeout=10000) + @Test(timeout = 10000) public void testMatchingRequestsForTopPriority() { - String[] hosts = { "host1" }; - String[] racks = { "rack1" }; + String[] hosts = {"host1"}; + String[] racks = {"rack1"}; AMRMClient.ContainerRequest req1 = new AMRMClient.ContainerRequest( - Resource.newInstance(2048, 1), hosts, racks, - Priority.newInstance(1)); + Resource.newInstance(2048, 1), hosts, racks, + Priority.newInstance(1)); AMRMClient.ContainerRequest req2 = new AMRMClient.ContainerRequest( - Resource.newInstance(1024, 1), hosts, racks, - Priority.newInstance(2)); + Resource.newInstance(1024, 1), hosts, racks, + Priority.newInstance(2)); AMRMClient.ContainerRequest req3 = new AMRMClient.ContainerRequest( - Resource.newInstance(1024, 1), hosts, racks, - Priority.newInstance(3)); + Resource.newInstance(1024, 1), hosts, racks, + Priority.newInstance(3)); amrmClient.addContainerRequest(req1); amrmClient.addContainerRequest(req2); amrmClient.addContainerRequest(req3); Assert.assertTrue(amrmClient.getMatchingRequestsForTopPriority("host1", - Resource.newInstance(1024, 1)).isEmpty()); + Resource.newInstance(1024, 1)).isEmpty()); List> ret = - amrmClient.getMatchingRequestsForTopPriority("host1", - Resource.newInstance(2048, 1)); + amrmClient.getMatchingRequestsForTopPriority("host1", + Resource.newInstance(2048, 1)); Assert.assertFalse(ret.isEmpty()); Assert.assertEquals(req1, ret.get(0).iterator().next()); @@ -93,5 +93,4 @@ public void testMatchingRequestsForTopPriority() { Assert.assertFalse(ret.isEmpty()); Assert.assertEquals(req2, ret.get(0).iterator().next()); } - } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java index 3061ceaa88..854db80ed6 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -97,9 +97,8 @@ import com.google.common.collect.Maps; - public class TestAMContainer { - @Test (timeout=5000) + @Test(timeout = 5000) // Assign before launch. public void tetSingleSuccessfulTaskFlow() { WrappedContainer wc = new WrappedContainer(); @@ -122,7 +121,7 @@ public void tetSingleSuccessfulTaskFlow() { assertEquals(wc.taskAttemptID, wc.amContainer.getCurrentTaskAttempt()); assertTrue(wc.amContainer.getCurrentTaskAttemptAllocationTime() > 0); assertTrue(wc.amContainer.getCurrentTaskAttemptAllocationTime() >= currTime); - + // Container Launched wc.containerLaunched(); wc.verifyState(AMContainerState.RUNNING); @@ -158,7 +157,7 @@ public void tetSingleSuccessfulTaskFlow() { assertFalse(wc.amContainer.isInErrorState()); } - @Test (timeout=5000) + @Test(timeout = 5000) // Assign after launch. public void testSingleSuccessfulTaskFlow2() { WrappedContainer wc = new WrappedContainer(); @@ -210,7 +209,7 @@ public void testSingleSuccessfulTaskFlow2() { assertFalse(wc.amContainer.isInErrorState()); } - @Test (timeout=5000) + @Test(timeout = 5000) // Assign before launch. public void tetMultipleSuccessfulTaskFlow() { WrappedContainer wc = new WrappedContainer(); @@ -282,7 +281,7 @@ public void tetMultipleSuccessfulTaskFlow() { assertFalse(wc.amContainer.isInErrorState()); } - @Test (timeout=5000) + @Test(timeout = 5000) public void testSingleSuccessfulTaskFlowStopRequest() { WrappedContainer wc = new WrappedContainer(); @@ -319,7 +318,7 @@ public void testSingleSuccessfulTaskFlowStopRequest() { assertFalse(wc.amContainer.isInErrorState()); } - @Test (timeout=5000) + @Test(timeout = 5000) public void testSingleSuccessfulTaskFlowFailedNMStopRequest() { WrappedContainer wc = new WrappedContainer(); @@ -359,7 +358,7 @@ public void testSingleSuccessfulTaskFlowFailedNMStopRequest() { } @SuppressWarnings("rawtypes") - @Test (timeout=5000) + @Test(timeout = 5000) public void testMultipleAllocationsWhileActive() { WrappedContainer wc = new WrappedContainer(); List outgoingEvents; @@ -399,7 +398,7 @@ public void testMultipleAllocationsWhileActive() { } @SuppressWarnings("rawtypes") - @Test (timeout=5000) + @Test(timeout = 5000) public void testMultipleAllocationsAtLaunching() { WrappedContainer wc = new WrappedContainer(); List outgoingEvents; @@ -439,7 +438,7 @@ public void testMultipleAllocationsAtLaunching() { } @SuppressWarnings("rawtypes") - @Test (timeout=5000) + @Test(timeout = 5000) public void testContainerTimedOutAtRunning() { WrappedContainer wc = new WrappedContainer(); List outgoingEvents; @@ -475,7 +474,7 @@ public void testContainerTimedOutAtRunning() { } @SuppressWarnings("rawtypes") - @Test (timeout=5000) + @Test(timeout = 5000) public void testStopRequestedAtRunning() { WrappedContainer wc = new WrappedContainer(); List outgoingEvents; @@ -511,7 +510,7 @@ public void testStopRequestedAtRunning() { } @SuppressWarnings("rawtypes") - @Test (timeout=5000) + @Test(timeout = 5000) public void testLaunchFailure() { WrappedContainer wc = new WrappedContainer(); List outgoingEvents; @@ -534,7 +533,7 @@ public void testLaunchFailure() { for (Event e : outgoingEvents) { if (e.getType() == TaskAttemptEventType.TA_CONTAINER_TERMINATING) { Assert.assertEquals(TaskAttemptTerminationCause.CONTAINER_LAUNCH_FAILED, - ((TaskAttemptEventContainerTerminating)e).getTerminationCause()); + ((TaskAttemptEventContainerTerminating) e).getTerminationCause()); } } @@ -547,7 +546,7 @@ public void testLaunchFailure() { assertFalse(wc.amContainer.isInErrorState()); } - @Test (timeout=5000) + @Test(timeout = 5000) public void testContainerCompletedAtAllocated() { WrappedContainer wc = new WrappedContainer(); wc.verifyState(AMContainerState.ALLOCATED); @@ -562,7 +561,7 @@ public void testContainerCompletedAtAllocated() { } @SuppressWarnings("rawtypes") - @Test (timeout=5000) + @Test(timeout = 5000) // Verify that incoming NM launched events to COMPLETED containers are // handled. public void testContainerCompletedAtLaunching() { @@ -571,7 +570,6 @@ public void testContainerCompletedAtLaunching() { wc.launchContainer(); - wc.assignTaskAttempt(wc.taskAttemptID); wc.containerCompleted(); @@ -584,7 +582,7 @@ public void testContainerCompletedAtLaunching() { TaskAttemptEventType.TA_CONTAINER_TERMINATED, AMNodeEventType.N_CONTAINER_COMPLETED); Assert.assertEquals(TaskAttemptTerminationCause.CONTAINER_LAUNCH_FAILED, - ((TaskAttemptEventContainerTerminated)outgoingEvents.get(0)).getTerminationCause()); + ((TaskAttemptEventContainerTerminated) outgoingEvents.get(0)).getTerminationCause()); assertFalse(wc.amContainer.isInErrorState()); @@ -596,7 +594,7 @@ public void testContainerCompletedAtLaunching() { } @SuppressWarnings("rawtypes") - @Test (timeout=5000) + @Test(timeout = 5000) public void testContainerCompletedAtLaunchingSpecificClusterError() { WrappedContainer wc = new WrappedContainer(); List outgoingEvents; @@ -615,7 +613,7 @@ public void testContainerCompletedAtLaunchingSpecificClusterError() { TaskAttemptEventType.TA_CONTAINER_TERMINATED_BY_SYSTEM, AMNodeEventType.N_CONTAINER_COMPLETED); Assert.assertEquals(TaskAttemptTerminationCause.NODE_DISK_ERROR, - ((TaskAttemptEventContainerTerminatedBySystem)outgoingEvents.get(0)).getTerminationCause()); + ((TaskAttemptEventContainerTerminatedBySystem) outgoingEvents.get(0)).getTerminationCause()); assertFalse(wc.amContainer.isInErrorState()); @@ -627,14 +625,13 @@ public void testContainerCompletedAtLaunchingSpecificClusterError() { } @SuppressWarnings("rawtypes") - @Test (timeout=5000) + @Test(timeout = 5000) public void testContainerCompletedAtLaunchingSpecificError() { WrappedContainer wc = new WrappedContainer(); List outgoingEvents; wc.launchContainer(); - wc.assignTaskAttempt(wc.taskAttemptID); wc.containerCompleted(ContainerExitStatus.ABORTED, TaskAttemptTerminationCause.NODE_FAILED, "NodeFailed"); @@ -648,7 +645,7 @@ public void testContainerCompletedAtLaunchingSpecificError() { TaskAttemptEventType.TA_CONTAINER_TERMINATED, AMNodeEventType.N_CONTAINER_COMPLETED); Assert.assertEquals(TaskAttemptTerminationCause.NODE_FAILED, - ((TaskAttemptEventContainerTerminated)outgoingEvents.get(0)).getTerminationCause()); + ((TaskAttemptEventContainerTerminated) outgoingEvents.get(0)).getTerminationCause()); assertFalse(wc.amContainer.isInErrorState()); @@ -660,7 +657,7 @@ public void testContainerCompletedAtLaunchingSpecificError() { } @SuppressWarnings("rawtypes") - @Test (timeout=5000) + @Test(timeout = 5000) public void testContainerCompletedAtIdle() { WrappedContainer wc = new WrappedContainer(); List outgoingEvents; @@ -689,7 +686,7 @@ public void testContainerCompletedAtIdle() { } @SuppressWarnings("rawtypes") - @Test (timeout=5000) + @Test(timeout = 5000) public void testContainerCompletedAtRunning() { WrappedContainer wc = new WrappedContainer(); List outgoingEvents; @@ -725,7 +722,7 @@ public void testContainerCompletedAtRunning() { } @SuppressWarnings("rawtypes") - @Test (timeout=5000) + @Test(timeout = 5000) public void testContainerPreemptedAtRunning() { WrappedContainer wc = new WrappedContainer(); List outgoingEvents; @@ -749,7 +746,7 @@ public void testContainerPreemptedAtRunning() { Event event = findEventByType(outgoingEvents, TaskAttemptEventType.TA_CONTAINER_TERMINATED_BY_SYSTEM); Assert.assertEquals(TaskAttemptTerminationCause.EXTERNAL_PREEMPTION, - ((TaskAttemptEventContainerTerminatedBySystem)event).getTerminationCause()); + ((TaskAttemptEventContainerTerminatedBySystem) event).getTerminationCause()); verifyUnOrderedOutgoingEventTypes(outgoingEvents, TaskAttemptEventType.TA_CONTAINER_TERMINATED_BY_SYSTEM, AMNodeEventType.N_CONTAINER_COMPLETED); @@ -767,7 +764,7 @@ public void testContainerPreemptedAtRunning() { } @SuppressWarnings("rawtypes") - @Test (timeout=5000) + @Test(timeout = 5000) public void testContainerInternallyPreemptedAtRunning() { WrappedContainer wc = new WrappedContainer(); List outgoingEvents; @@ -792,7 +789,7 @@ public void testContainerInternallyPreemptedAtRunning() { TaskAttemptEventType.TA_CONTAINER_TERMINATED_BY_SYSTEM, AMNodeEventType.N_CONTAINER_COMPLETED); Assert.assertEquals(TaskAttemptTerminationCause.INTERNAL_PREEMPTION, - ((TaskAttemptEventContainerTerminatedBySystem)outgoingEvents.get(0)).getTerminationCause()); + ((TaskAttemptEventContainerTerminatedBySystem) outgoingEvents.get(0)).getTerminationCause()); assertFalse(wc.amContainer.isInErrorState()); @@ -807,7 +804,7 @@ public void testContainerInternallyPreemptedAtRunning() { } @SuppressWarnings("rawtypes") - @Test (timeout=5000) + @Test(timeout = 5000) public void testContainerDiskFailedAtRunning() { WrappedContainer wc = new WrappedContainer(); List outgoingEvents; @@ -829,7 +826,7 @@ public void testContainerDiskFailedAtRunning() { outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2); Event event = findEventByType(outgoingEvents, TaskAttemptEventType.TA_CONTAINER_TERMINATED_BY_SYSTEM); Assert.assertEquals(TaskAttemptTerminationCause.NODE_DISK_ERROR, - ((TaskAttemptEventContainerTerminatedBySystem)event).getTerminationCause()); + ((TaskAttemptEventContainerTerminatedBySystem) event).getTerminationCause()); verifyUnOrderedOutgoingEventTypes(outgoingEvents, TaskAttemptEventType.TA_CONTAINER_TERMINATED_BY_SYSTEM, AMNodeEventType.N_CONTAINER_COMPLETED); @@ -847,7 +844,7 @@ public void testContainerDiskFailedAtRunning() { } @SuppressWarnings("rawtypes") - @Test (timeout=5000) + @Test(timeout = 5000) public void testTaskAssignedToCompletedContainer() { WrappedContainer wc = new WrappedContainer(); List outgoingEvents; @@ -878,7 +875,7 @@ public void testTaskAssignedToCompletedContainer() { } @SuppressWarnings("rawtypes") - @Test (timeout=5000) + @Test(timeout = 5000) public void testNodeFailedAtRunning() { WrappedContainer wc = new WrappedContainer(); List outgoingEvents; @@ -915,7 +912,7 @@ public void testNodeFailedAtRunning() { } @SuppressWarnings("rawtypes") - @Test (timeout=5000) + @Test(timeout = 5000) public void testNodeFailedAtIdleMultipleAttempts() { WrappedContainer wc = new WrappedContainer(); List outgoingEvents; @@ -959,7 +956,7 @@ public void testNodeFailedAtIdleMultipleAttempts() { } @SuppressWarnings("rawtypes") - @Test (timeout=5000) + @Test(timeout = 5000) public void testNodeFailedAtRunningMultipleAttempts() { WrappedContainer wc = new WrappedContainer(); List outgoingEvents; @@ -1003,7 +1000,7 @@ public void testNodeFailedAtRunningMultipleAttempts() { } @SuppressWarnings("rawtypes") - @Test (timeout=5000) + @Test(timeout = 5000) public void testNodeFailedAtCompletedMultipleSuccessfulTAs() { WrappedContainer wc = new WrappedContainer(); List outgoingEvents; @@ -1031,7 +1028,7 @@ public void testNodeFailedAtCompletedMultipleSuccessfulTAs() { assertEquals(2, wc.amContainer.getAllTaskAttempts().size()); } - @Test (timeout=5000) + @Test(timeout = 5000) public void testDuplicateCompletedEvents() { WrappedContainer wc = new WrappedContainer(); @@ -1055,7 +1052,7 @@ public void testDuplicateCompletedEvents() { wc.verifyHistoryStopEvent(); } - @Test (timeout=5000) + @Test(timeout = 5000) public void testLocalResourceAddition() { WrappedContainer wc = new WrappedContainer(); @@ -1118,7 +1115,7 @@ public void testLocalResourceAddition() { } @SuppressWarnings("unchecked") - @Test (timeout=5000) + @Test(timeout = 5000) public void testCredentialsTransfer() { WrappedContainerMultipleDAGs wc = new WrappedContainerMultipleDAGs(); @@ -1194,7 +1191,7 @@ public void testCredentialsTransfer() { // Move to running a third DAG, with Credentials this time wc.setNewDAGID(dagID3); - wc.assignTaskAttempt(attempt31, LRs , dag3Credentials); + wc.assignTaskAttempt(attempt31, LRs, dag3Credentials); argumentCaptor = ArgumentCaptor.forClass(AMContainerTask.class); verify(wc.tal, times(5)).registerTaskAttempt(argumentCaptor.capture(), eq(wc.containerID), eq(0)); fetchedTask = argumentCaptor.getAllValues().get(4); @@ -1234,7 +1231,7 @@ static class WrappedContainer { EventHandler eventHandler; AppContext appContext; - + HistoryEventHandler historyEventHandler; TezDAGID dagID; @@ -1273,14 +1270,14 @@ public WrappedContainer(boolean shouldProfile, String profileString, int cIdInt) vertexID = TezVertexID.getInstance(dagID, 1); taskID = TezTaskID.getInstance(vertexID, 1); taskAttemptID = TezTaskAttemptID.getInstance(taskID, 1); - + eventHandler = mock(EventHandler.class); historyEventHandler = mock(HistoryEventHandler.class); Configuration conf = new Configuration(false); appContext = mock(AppContext.class); doReturn(new HashMap()).when(appContext) - .getApplicationACLs(); + .getApplicationACLs(); doReturn(eventHandler).when(appContext).getEventHandler(); doReturn(appAttemptID).when(appContext).getApplicationAttemptId(); doReturn(applicationID).when(appContext).getApplicationID(); @@ -1321,13 +1318,13 @@ public void verifyNoOutgoingEvents() { * * @return a list of outgoing events from the AMContainer. */ - @SuppressWarnings({ "rawtypes", "unchecked" }) + @SuppressWarnings({"rawtypes", "unchecked"}) public List verifyCountAndGetOutgoingEvents(int invocations) { ArgumentCaptor args = ArgumentCaptor.forClass(Event.class); verify(eventHandler, times(invocations)).handle(args.capture()); return args.getAllValues(); } - + public void verifyHistoryStopEvent() { ArgumentCaptor args = ArgumentCaptor.forClass(DAGHistoryEvent.class); verify(historyEventHandler, times(1)).handle(args.capture()); @@ -1353,7 +1350,7 @@ public void assignTaskAttempt(TezTaskAttemptID taID) { } public void assignTaskAttempt(TezTaskAttemptID taID, - Map additionalResources, Credentials credentials) { + Map additionalResources, Credentials credentials) { reset(eventHandler); doReturn(taID).when(taskSpec).getTaskAttemptID(); amContainer.handle(new AMContainerEventAssignTA(containerID, taID, taskSpec, @@ -1424,11 +1421,11 @@ public void verifyState(AMContainerState state) { state, amContainer.getState()); } } - + private static class WrappedContainerMultipleDAGs extends WrappedContainer { - + private TezDAGID newDAGID = null; - + @Override protected void mockDAGID() { doAnswer(new Answer() { @@ -1438,7 +1435,7 @@ public TezDAGID answer(InvocationOnMock invocation) throws Throwable { } }).when(appContext).getCurrentDAGID(); } - + void setNewDAGID(TezDAGID newDAGID) { this.newDAGID = newDAGID; } @@ -1446,7 +1443,7 @@ void setNewDAGID(TezDAGID newDAGID) { @SuppressWarnings("rawtypes") private void verifyUnOrderedOutgoingEventTypes(List events, - Enum... expectedTypes) { + Enum... expectedTypes) { List> expectedTypeList = new LinkedList>(); for (Enum expectedType : expectedTypes) { @@ -1481,7 +1478,7 @@ private Event findEventByType(List events, Enum type) { } return null; } - + private LocalResource createLocalResource(String name) { LocalResource lr = LocalResource.newInstance(URL.newInstance(null, "localhost", 2321, name), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, 1, 1000000); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainerMap.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainerMap.java index 17a9c532f2..5fac834772 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainerMap.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainerMap.java @@ -45,12 +45,12 @@ import org.apache.tez.dag.app.TaskCommunicatorManagerInterface; import org.apache.tez.serviceplugins.api.ServicePluginException; import org.junit.Test; + import static org.mockito.Mockito.when; public class TestAMContainerMap { - - @Test (timeout = 10000) + @Test(timeout = 10000) public void testCleanupOnDagComplete() { ContainerHeartbeatHandler chh = mock(ContainerHeartbeatHandler.class); @@ -58,11 +58,9 @@ public void testCleanupOnDagComplete() { AppContext appContext = mock(AppContext.class); when(appContext.getAMConf()).thenReturn(new Configuration()); - - int numContainers = 7; WrappedContainer[] wContainers = new WrappedContainer[numContainers]; - for (int i = 0 ; i < numContainers ; i++) { + for (int i = 0; i < numContainers; i++) { WrappedContainer wc = new WrappedContainer(false, null, i); wContainers[i] = wc; @@ -71,11 +69,10 @@ public void testCleanupOnDagComplete() { AMContainerMap amContainerMap = new AMContainerMapForTest(chh, tal, mock( ContainerSignatureMatcher.class), appContext, wContainers); - for (int i = 0 ; i < numContainers ; i++) { + for (int i = 0; i < numContainers; i++) { amContainerMap.addContainerIfNew(wContainers[i].container, 0, 0, 0); } - // Container 1 in LAUNCHING state wContainers[0].launchContainer(); wContainers[0].verifyState(AMContainerState.LAUNCHING); @@ -128,7 +125,6 @@ public void testCleanupOnDagComplete() { private static class AMContainerMapForTest extends AMContainerMap { - private WrappedContainer[] wrappedContainers; public AMContainerMapForTest(ContainerHeartbeatHandler chh, @@ -148,6 +144,5 @@ AMContainer createAmContainer(Container container, int launcherId, int taskCommId, String auxiliaryService) { return wrappedContainers[container.getId().getId()].amContainer; } - } } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/node/TestAMNodeTracker.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/node/TestAMNodeTracker.java index 060cdc4dcd..355528d7a9 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/node/TestAMNodeTracker.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/node/TestAMNodeTracker.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -56,14 +56,14 @@ import com.google.common.collect.Lists; -@SuppressWarnings({ "resource", "rawtypes" }) +@SuppressWarnings({"resource", "rawtypes"}) public class TestAMNodeTracker { private static final Logger LOG = LoggerFactory.getLogger(TestAMNodeTracker.class); DrainDispatcher dispatcher; EventHandler eventHandler; - + @Before public void setup() { dispatcher = new DrainDispatcher(); @@ -71,9 +71,10 @@ public void setup() { dispatcher.start(); eventHandler = dispatcher.getEventHandler(); } - - class TestEventHandler implements EventHandler{ + + class TestEventHandler implements EventHandler { List events = Lists.newLinkedList(); + @SuppressWarnings("unchecked") @Override public void handle(Event event) { @@ -81,13 +82,13 @@ public void handle(Event event) { eventHandler.handle(event); } } - + @After public void teardown() { dispatcher.stop(); } - - @Test(timeout=5000) + + @Test(timeout = 5000) public void testHealthUpdateKnownNode() { AppContext appContext = mock(AppContext.class); @@ -106,7 +107,7 @@ public void testHealthUpdateKnownNode() { amNodeTracker.stop(); } - @Test(timeout=5000) + @Test(timeout = 5000) public void testHealthUpdateUnknownNode() { AppContext appContext = mock(AppContext.class); @@ -126,7 +127,7 @@ public void testHealthUpdateUnknownNode() { // the log message for verification. } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testMultipleSourcesNodeRegistration() { AppContext appContext = mock(AppContext.class); AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext); @@ -149,7 +150,7 @@ public void testMultipleSourcesNodeRegistration() { assertNotNull(amNodeTracker.get(nodeId2, 1)); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testMultipleSourcesNodeCountUpdated() { AppContext appContext = mock(AppContext.class); AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext); @@ -175,7 +176,7 @@ public void testMultipleSourcesNodeCountUpdated() { assertNotNull(amNodeTracker.get(nodeId2, 1)); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testSingleNodeNotBlacklisted() { AppContext appContext = mock(AppContext.class); Configuration conf = new Configuration(false); @@ -198,7 +199,7 @@ public void testSingleNodeNotBlacklisted() { _testSingleNodeNotBlacklisted(amNodeTracker, handler, 0); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testSingleNodeNotBlacklistedAlternateScheduler() { AppContext appContext = mock(AppContext.class); Configuration conf = new Configuration(false); @@ -221,7 +222,7 @@ public void testSingleNodeNotBlacklistedAlternateScheduler() { _testSingleNodeNotBlacklisted(amNodeTracker, handler, 1); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testSingleNodeNotBlacklistedAlternateScheduler2() { AppContext appContext = mock(AppContext.class); Configuration conf = new Configuration(false); @@ -243,7 +244,7 @@ public void testSingleNodeNotBlacklistedAlternateScheduler2() { // Register multiple nodes from a scheduler which isn't being tested. // This should not affect the blacklisting behaviour - for (int i = 0 ; i < 10 ; i++) { + for (int i = 0; i < 10; i++) { amNodeTracker.nodeSeen(NodeId.newInstance("fakenode" + i, 3333), 0); } @@ -252,7 +253,7 @@ public void testSingleNodeNotBlacklistedAlternateScheduler2() { assertFalse(amNodeTracker.isBlacklistingIgnored(0)); } - @Test(timeout=10000) + @Test(timeout = 10000) public void testNodeSelfBlacklist() { AppContext appContext = mock(AppContext.class); Configuration conf = new Configuration(false); @@ -275,7 +276,7 @@ public void testNodeSelfBlacklist() { } } - @Test(timeout=10000) + @Test(timeout = 10000) public void testNodeSelfBlacklistAlternateScheduler1() { AppContext appContext = mock(AppContext.class); Configuration conf = new Configuration(false); @@ -298,7 +299,7 @@ public void testNodeSelfBlacklistAlternateScheduler1() { } } - @Test(timeout=10000) + @Test(timeout = 10000) public void testNodeSelfBlacklistAlternateScheduler2() { AppContext appContext = mock(AppContext.class); Configuration conf = new Configuration(false); @@ -317,7 +318,7 @@ public void testNodeSelfBlacklistAlternateScheduler2() { try { // Register multiple nodes from a scheduler which isn't being tested. // This should not affect the blacklisting behaviour - for (int i = 0 ; i < 100 ; i++) { + for (int i = 0; i < 100; i++) { amNodeTracker.nodeSeen(NodeId.newInstance("fakenode" + i, 3333), 0); } _testNodeSelfBlacklist(amNodeTracker, handler, 1); @@ -327,7 +328,7 @@ public void testNodeSelfBlacklistAlternateScheduler2() { } } - @Test(timeout=10000) + @Test(timeout = 10000) public void testMultipleAMNodeIDs() { AppContext appContext = mock(AppContext.class); Configuration conf = new Configuration(false); @@ -337,7 +338,7 @@ public void testMultipleAMNodeIDs() { doReturn(amNodeTracker).when(appContext).getNodeTracker(); AMContainerMap amContainerMap = mock(AMContainerMap.class); TaskSchedulerManager taskSchedulerManager = - mock(TaskSchedulerManager.class); + mock(TaskSchedulerManager.class); dispatcher.register(AMNodeEventType.class, amNodeTracker); dispatcher.register(AMContainerEventType.class, amContainerMap); dispatcher.register(AMSchedulerEventType.class, taskSchedulerManager); @@ -391,7 +392,6 @@ public void testNodeCompletedAndCleanup() { amNodeTracker.dagComplete(mock(DAG.class)); assertEquals(5, amNode.getContainers().size()); - // Mark 2 as complete. Finish 2nd dag. for (int i = 0; i < 2; i++) { amNodeTracker.handle( @@ -419,30 +419,27 @@ public void testNodeCompletedAndCleanup() { assertEquals(5, amNode.getContainers().size()); amNodeTracker.dagComplete(mock(DAG.class)); assertEquals(0, amNode.getContainers().size()); - } finally { amNodeTracker.stop(); } - } - @Test(timeout=10000) + @Test(timeout = 10000) public void testNodeUnhealthyRescheduleTasksEnabled() throws Exception { _testNodeUnhealthyRescheduleTasks(true, false); } - @Test(timeout=10000) + @Test(timeout = 10000) public void testNodeUnhealthyRescheduleTasksDisabled() throws Exception { _testNodeUnhealthyRescheduleTasks(false, false); } - - @Test(timeout=10000) + @Test(timeout = 10000) public void testNodeUnhealthyRescheduleTasksEnabledAMNode() throws Exception { _testNodeUnhealthyRescheduleTasks(true, true); } - @Test(timeout=10000) + @Test(timeout = 10000) public void testNodeUnhealthyRescheduleTasksDisabledAMNode() throws Exception { _testNodeUnhealthyRescheduleTasks(false, true); } @@ -569,14 +566,13 @@ private void _testNodeSelfBlacklist(AMNodeTracker amNodeTracker, TestEventHandle assertEquals(4, handler.events.size()); assertEquals(AMContainerEventType.C_NODE_FAILED, handler.events.get(0).getType()); - assertEquals(cId1, ((AMContainerEventNodeFailed)handler.events.get(0)).getContainerId()); + assertEquals(cId1, ((AMContainerEventNodeFailed) handler.events.get(0)).getContainerId()); assertEquals(AMContainerEventType.C_NODE_FAILED, handler.events.get(1).getType()); - assertEquals(cId2, ((AMContainerEventNodeFailed)handler.events.get(1)).getContainerId()); + assertEquals(cId2, ((AMContainerEventNodeFailed) handler.events.get(1)).getContainerId()); assertEquals(AMContainerEventType.C_NODE_FAILED, handler.events.get(2).getType()); - assertEquals(cId3, ((AMContainerEventNodeFailed)handler.events.get(2)).getContainerId()); + assertEquals(cId3, ((AMContainerEventNodeFailed) handler.events.get(2)).getContainerId()); assertEquals(AMSchedulerEventType.S_NODE_BLACKLISTED, handler.events.get(3).getType()); - assertEquals(node.getNodeId(), ((AMSchedulerEventNodeBlacklistUpdate)handler.events.get(3)).getNodeId()); - + assertEquals(node.getNodeId(), ((AMSchedulerEventNodeBlacklistUpdate) handler.events.get(3)).getNodeId()); // Trigger one more node failure, which should cause BLACKLISTING to be disabled ContainerId cId4 = mock(ContainerId.class); @@ -619,7 +615,6 @@ private void _testNodeSelfBlacklist(AMNodeTracker amNodeTracker, TestEventHandle // drain all previous events dispatcher.await(); - // Increase the number of nodes. BLACKLISTING should be re-enabled. // Node 1 and Node 2 should go into BLACKLISTED state handler.events.clear(); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/web/TestAMWebController.java b/tez-dag/src/test/java/org/apache/tez/dag/app/web/TestAMWebController.java index ced9ca0abe..34b730ad04 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/app/web/TestAMWebController.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/app/web/TestAMWebController.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -145,7 +145,7 @@ public void testCorsHeadersAreSet() { "X-Requested-With,Content-Type,Accept,Origin"); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void sendErrorResponseIfNoAccess() throws Exception { AMWebController amWebController = new AMWebController(mockRequestContext, mockAppContext, "TEST_HISTORY_URL"); @@ -172,7 +172,7 @@ public void sendErrorResponseIfNoAccess() throws Exception { @Captor ArgumentCaptor> singleResultCaptor; - @Test (timeout = 5000) + @Test(timeout = 5000) public void testDagProgressResponse() { AMWebController amWebController = new AMWebController(mockRequestContext, mockAppContext, "TEST_HISTORY_URL"); @@ -198,7 +198,7 @@ public void testDagProgressResponse() { Assert.assertEquals(66.0, progressInfo.getProgress(), 0.1); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testVertexProgressResponse() { AMWebController amWebController = new AMWebController(mockRequestContext, mockAppContext, "TEST_HISTORY_URL"); @@ -229,7 +229,7 @@ public void testVertexProgressResponse() { Assert.assertEquals(66.0f, progressInfo.getProgress(), 0.1); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testHasAccessWithAclsDisabled() { Configuration conf = new Configuration(false); conf.setBoolean(TezConfiguration.TEZ_AM_ACLS_ENABLED, false); @@ -244,7 +244,7 @@ public void testHasAccessWithAclsDisabled() { Assert.assertEquals(true, AMWebController._hasAccess(mockUser, mockAppContext)); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testHasAccess() { Configuration conf = new Configuration(false); conf.setBoolean(TezConfiguration.TEZ_AM_ACLS_ENABLED, true); @@ -263,11 +263,10 @@ public void testHasAccess() { Assert.assertEquals(true, AMWebController._hasAccess(testUser, mockAppContext)); } - // AM Webservice Version 2 //ArgumentCaptor> returnResultCaptor; @Captor - ArgumentCaptor> returnResultCaptor; + ArgumentCaptor> returnResultCaptor; @SuppressWarnings("unchecked") @Test(timeout = 5000) @@ -277,7 +276,6 @@ public void testGetDagInfo() { AMWebController spy = spy(amWebController); DAG mockDAG = mock(DAG.class); - doReturn(TezDAGID.fromString("dag_1422960590892_0007_42")).when(mockDAG).getID(); doReturn(66.0f).when(mockDAG).getCompletedTaskProgress(); doReturn(DAGState.RUNNING).when(mockDAG).getState(); @@ -317,7 +315,6 @@ public void testGetDagInfo() { Assert.assertEquals("66.0", dagInfo.get("progress")); Assert.assertEquals("RUNNING", dagInfo.get("status")); Assert.assertNotNull(dagInfo.get("counters")); - } @SuppressWarnings("unchecked") @@ -443,7 +440,6 @@ private Vertex createMockVertex(String vertexIDStr, VertexState status, float pr return mockVertex; } - private void verifySingleVertexResult(Vertex mockVertex2, Map vertex2Result) { ProgressBuilder progress; Assert.assertEquals(mockVertex2.getVertexId().toString(), vertex2Result.get("id")); @@ -479,9 +475,9 @@ private void verifySingleVertexResult(Vertex mockVertex2, Map ve @SuppressWarnings("unchecked") @Test(timeout = 5000) public void testGetTasksInfoWithTaskIds() { - List tasks = createMockTasks(); - List vertexMinIds = Arrays.asList(); - List > taskMinIds = Arrays.asList(Arrays.asList(0, 0), + List tasks = createMockTasks(); + List vertexMinIds = Arrays.asList(); + List> taskMinIds = Arrays.asList(Arrays.asList(0, 0), Arrays.asList(0, 3), Arrays.asList(0, 1)); @@ -516,9 +512,9 @@ public void testGetTasksInfoWithTaskIds() { @SuppressWarnings("unchecked") @Test(timeout = 5000) public void testGetTasksInfoGracefulTaskFetch() { - List tasks = createMockTasks(); - List vertexMinIds = Arrays.asList(); - List > taskMinIds = Arrays.asList(Arrays.asList(0, 0), + List tasks = createMockTasks(); + List vertexMinIds = Arrays.asList(); + List> taskMinIds = Arrays.asList(Arrays.asList(0, 0), Arrays.asList(0, 6), Arrays.asList(0, 1)); @@ -540,9 +536,9 @@ public void testGetTasksInfoGracefulTaskFetch() { @SuppressWarnings("unchecked") @Test(timeout = 5000) public void testGetTasksInfoWithVertexId() { - List tasks = createMockTasks(); - List vertexMinIds = Arrays.asList(0); - List > taskMinIds = Arrays.asList(); + List tasks = createMockTasks(); + List vertexMinIds = Arrays.asList(0); + List> taskMinIds = Arrays.asList(); Map result = getTasksTestHelper(tasks, taskMinIds, vertexMinIds, AMWebController.MAX_QUERIED); @@ -573,9 +569,9 @@ public void testGetTasksInfoWithVertexId() { @SuppressWarnings("unchecked") @Test(timeout = 5000) public void testGetTasksInfoWithJustDAGId() { - List tasks = createMockTasks(); - List vertexMinIds = Arrays.asList(); - List > taskMinIds = Arrays.asList(); + List tasks = createMockTasks(); + List vertexMinIds = Arrays.asList(); + List> taskMinIds = Arrays.asList(); Map result = getTasksTestHelper(tasks, taskMinIds, vertexMinIds, AMWebController.MAX_QUERIED); @@ -621,7 +617,7 @@ public int compare(Map first, Map second) { Collections.sort(list, new MapComparator(propertyName)); } - Map getTasksTestHelper(List tasks, List > taskMinIds, + Map getTasksTestHelper(List tasks, List> taskMinIds, List vertexMinIds, Integer limit) { //Creating mock DAG DAG mockDAG = mock(DAG.class); @@ -639,7 +635,7 @@ Map getTasksTestHelper(List tasks, List > t //Creating mock tasks and attaching to mock vertex Map taskMap = Maps.newHashMap(); - for(Task task : tasks) { + for (Task task : tasks) { TezTaskID taskId = task.getTaskID(); int taskIndex = taskId.getId(); doReturn(task).when(mockVertex).getTask(taskIndex); @@ -681,7 +677,7 @@ private List createMockTasks() { Task mockTask4 = createMockTask("task_1441301219877_0109_1_00_000003", TaskState.SUCCEEDED, .8f); - List tasks = Arrays.asList(mockTask1, mockTask2, mockTask3, mockTask4); + List tasks = Arrays.asList(mockTask1, mockTask2, mockTask3, mockTask4); return tasks; } @@ -721,10 +717,10 @@ private void verifySingleTaskResult(Task mockTask, Map taskResul @SuppressWarnings("unchecked") @Test(timeout = 5000) public void testGetAttemptsInfoWithIds() { - List attempts = createMockAttempts(); - List vertexMinIds = Arrays.asList(); - List taskMinIds = Arrays.asList(); - List > attemptMinIds = Arrays.asList(Arrays.asList(0, 0, 0), + List attempts = createMockAttempts(); + List vertexMinIds = Arrays.asList(); + List taskMinIds = Arrays.asList(); + List> attemptMinIds = Arrays.asList(Arrays.asList(0, 0, 0), Arrays.asList(0, 0, 1), Arrays.asList(0, 0, 2), Arrays.asList(0, 0, 3)); @@ -758,8 +754,8 @@ public void testGetAttemptsInfoWithIds() { verifySingleAttemptResult(attempts.get(1), attemptsInfo.get(1)); } - Map getAttemptsTestHelper(List attempts, List > attemptMinIds, - List vertexMinIds, List taskMinIds, Integer limit) { + Map getAttemptsTestHelper(List attempts, List> attemptMinIds, + List vertexMinIds, List taskMinIds, Integer limit) { //Creating mock DAG DAG mockDAG = mock(DAG.class); doReturn(TezDAGID.fromString("dag_1441301219877_0109_1")).when(mockDAG).getID(); @@ -786,7 +782,7 @@ Map getAttemptsTestHelper(List attempts, List attemptsMap = Maps.newHashMap(); - for(TaskAttempt attempt : attempts) { + for (TaskAttempt attempt : attempts) { TezTaskAttemptID attemptId = attempt.getTaskAttemptID(); doReturn(attempt).when(mockTask).getAttempt(attemptId); attemptsMap.put(attemptId, attempt); @@ -828,7 +824,7 @@ private List createMockAttempts() { TaskAttempt mockAttempt4 = createMockAttempt("attempt_1441301219877_0109_1_00_000000_3", TaskAttemptState.SUCCEEDED, .8f); - List attempts = Arrays.asList(mockAttempt1, mockAttempt2, mockAttempt3, mockAttempt4); + List attempts = Arrays.asList(mockAttempt1, mockAttempt2, mockAttempt3, mockAttempt4); return attempts; } @@ -862,5 +858,4 @@ private void verifySingleAttemptResult(TaskAttempt mockTask, Map Assert.assertEquals(mockTask.getState().toString(), taskResult.get("status")); Assert.assertEquals(Float.toString(mockTask.getProgress()), taskResult.get("progress")); } - } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/TestHistoryEventHandler.java b/tez-dag/src/test/java/org/apache/tez/dag/history/TestHistoryEventHandler.java index 5a71a426ee..08fcabd506 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/history/TestHistoryEventHandler.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/history/TestHistoryEventHandler.java @@ -156,7 +156,7 @@ private void testLogLevel(HistoryLogLevel level, int expectedCount) { } private void testLogLevel(HistoryLogLevel defaultLogLevel, HistoryLogLevel dagLogLevel, - int expectedCount) { + int expectedCount) { HistoryEventHandler handler = createHandler(defaultLogLevel); InMemoryHistoryLoggingService.events.clear(); TezDAGID dagId1 = TezDAGID.getInstance(appId, 1); @@ -178,7 +178,9 @@ public static class InMemoryHistoryLoggingService extends HistoryLoggingService public InMemoryHistoryLoggingService() { super("InMemoryHistoryLoggingService"); } + static List events = new ArrayList<>(); + @Override public void handle(DAGHistoryEvent event) { events.add(event); @@ -199,11 +201,12 @@ private HistoryEventHandler createHandler(HistoryLogLevel logLevel) { AppContext appContext = mock(AppContext.class); when(appContext.getApplicationID()).thenReturn(appId); - when(appContext.getHadoopShim()).thenReturn(new HadoopShim() {}); + when(appContext.getHadoopShim()).thenReturn(new HadoopShim() { + }); when(appContext.getAMConf()).thenReturn(conf); when(appContext.getCurrentDAG()).thenReturn(dag); - HistoryEventHandler handler = new HistoryEventHandler(appContext); + HistoryEventHandler handler = new HistoryEventHandler(appContext); handler.init(conf); return handler; diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/TestHistoryEventType.java b/tez-dag/src/test/java/org/apache/tez/dag/history/TestHistoryEventType.java index c37442e189..240d9aaffa 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/history/TestHistoryEventType.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/history/TestHistoryEventType.java @@ -35,5 +35,4 @@ public void testDAGSpecificEventCheck() { } } } - } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java b/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java index 51b4bf9f92..f597857db7 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java @@ -6,9 +6,9 @@ * to you under the Apache License: Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing: software * distributed under the License is distributed on an "AS IS" BASIS: * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND: either express or implied. @@ -81,13 +81,11 @@ import java.util.List; import java.util.Map; - public class TestHistoryEventsProtoConversion { private static final Logger LOG = LoggerFactory.getLogger( TestHistoryEventsProtoConversion.class); - private HistoryEvent testProtoConversion(HistoryEvent event) throws IOException, TezException { ByteArrayOutputStream os = new ByteArrayOutputStream(); HistoryEvent deserializedEvent = null; @@ -122,12 +120,12 @@ private HistoryEvent testSummaryProtoConversion(HistoryEvent historyEvent) new ByteArrayInputStream(os.toByteArray())); deserializedEvent = ReflectionUtils.createClazzInstance( event.getClass().getName()); - ((SummaryEvent)deserializedEvent).fromSummaryProtoStream(summaryEventProto); + ((SummaryEvent) deserializedEvent).fromSummaryProtoStream(summaryEventProto); return deserializedEvent; } private void logEvents(HistoryEvent event, - HistoryEvent deserializedEvent) { + HistoryEvent deserializedEvent) { LOG.info("Initial Event toString: " + event.toString()); LOG.info("Deserialized Event toString: " + deserializedEvent.toString()); } @@ -143,7 +141,6 @@ private void testAppLaunchedEvent() throws Exception { } LOG.info("Initial Event toString: " + event.toString()); - } private void testAMLaunchedEvent() throws Exception { @@ -176,6 +173,7 @@ private void testAMStartedEvent() throws Exception { } private final String QUEUE_NAME = "TEST_QUEUE_NAME"; + private void testDAGSubmittedEvent() throws Exception { DAGSubmittedEvent event = new DAGSubmittedEvent(TezDAGID.getInstance( ApplicationId.newInstance(0, 1), 1), 1001l, @@ -223,8 +221,8 @@ private void testDAGStartedEvent() throws Exception { } private void testDAGKillRequestEvent() throws Exception { - DAGKillRequestEvent event = - new DAGKillRequestEvent(TezDAGID.getInstance(ApplicationId.newInstance(0, 1), 1), 100334l,false); + DAGKillRequestEvent event = + new DAGKillRequestEvent(TezDAGID.getInstance(ApplicationId.newInstance(0, 1), 1), 100334l, false); DAGKillRequestEvent deserializedEvent = (DAGKillRequestEvent) testProtoConversion(event); Assert.assertEquals(event.getDagID(), @@ -328,17 +326,17 @@ private void testVertexReconfigureDoneEvent() throws Exception { Map rootInputSpecUpdates = new HashMap(); rootInputSpecUpdates.put("input1", rootInputSpecUpdateBulk); rootInputSpecUpdates.put("input2", rootInputSpecUpdatePerTask); - + Map sourceEdgeManagers - = new HashMap(); + = new HashMap(); // add standard and custom edge - sourceEdgeManagers.put("foo", EdgeProperty.create(DataMovementType.SCATTER_GATHER, - DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, + sourceEdgeManagers.put("foo", EdgeProperty.create(DataMovementType.SCATTER_GATHER, + DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out1"), InputDescriptor.create("in1"))); sourceEdgeManagers.put("foo1", EdgeProperty.create(EdgeManagerPluginDescriptor.create("bar1") - .setUserPayload( - UserPayload.create(ByteBuffer.wrap(new String("payload").getBytes()), 100)), - DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, + .setUserPayload( + UserPayload.create(ByteBuffer.wrap(new String("payload").getBytes()), 100)), + DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out1"), InputDescriptor.create("in1"))); final long reconfigureDoneTime = 100; @@ -352,7 +350,7 @@ private void testVertexReconfigureDoneEvent() throws Exception { Assert.assertEquals(numTasks, event.getNumTasks()); Assert.assertEquals(reconfigureDoneTime, event.getReconfigureDoneTime()); VertexConfigurationDoneEvent deserializedEvent = (VertexConfigurationDoneEvent) - testProtoConversion(event); + testProtoConversion(event); Assert.assertEquals(event.getVertexID(), deserializedEvent.getVertexID()); Assert.assertEquals(event.getNumTasks(), deserializedEvent.getNumTasks()); Assert.assertEquals(event.isSetParallelismCalled(), deserializedEvent.isSetParallelismCalled()); @@ -489,7 +487,7 @@ private void testTaskAttemptStartedEvent() throws Exception { ApplicationAttemptId.newInstance( ApplicationId.newInstance(0, 1), 1), 1001), NodeId.newInstance( "host1", 19999), "inProgress", "Completed", "nodeHttpAddress" - ); + ); TaskAttemptStartedEvent deserializedEvent = (TaskAttemptStartedEvent) testProtoConversion(event); Assert.assertEquals(event.getTaskAttemptID(), @@ -512,9 +510,9 @@ private void testTaskAttemptFinishedEvent() throws Exception { TezTaskAttemptID.getInstance(TezTaskID.getInstance(TezVertexID.getInstance( TezDAGID.getInstance(ApplicationId.newInstance(0, 1), 1), 111), 1), 0), 1024, ContainerId.newInstance( - ApplicationAttemptId.newInstance( - ApplicationId.newInstance(0, 1), 1), 1001), NodeId.newInstance( - "host1", 19999), "inProgress", "Completed", "nodeHttpAddress"); + ApplicationAttemptId.newInstance( + ApplicationId.newInstance(0, 1), 1), 1001), NodeId.newInstance( + "host1", 19999), "inProgress", "Completed", "nodeHttpAddress"); TaskAttemptFinishedEvent deserializedEvent = (TaskAttemptFinishedEvent) testProtoConversion(event); Assert.assertEquals(event.getTaskAttemptID(), @@ -561,7 +559,7 @@ private void testTaskAttemptFinishedEvent() throws Exception { ContainerId.newInstance( ApplicationAttemptId.newInstance( ApplicationId.newInstance(0, 1), 1), 1001), NodeId.newInstance( - "host1", 19999), "inProgress", "Completed", "nodeHttpAddress"); + "host1", 19999), "inProgress", "Completed", "nodeHttpAddress"); TaskAttemptFinishedEvent deserializedEvent = (TaskAttemptFinishedEvent) testProtoConversion(event); Assert.assertEquals(event.getTaskAttemptID(), @@ -637,19 +635,19 @@ private void testTaskAttemptFinishedEvent() throws Exception { { // toString shouldn't include null fields TezTaskAttemptID taId = - TezTaskAttemptID.getInstance(TezTaskID.getInstance(TezVertexID.getInstance( - TezDAGID.getInstance(ApplicationId.newInstance(0, 1), 1), 111), 0), 0); + TezTaskAttemptID.getInstance(TezTaskID.getInstance(TezVertexID.getInstance( + TezDAGID.getInstance(ApplicationId.newInstance(0, 1), 1), 111), 0), 0); long timestamp = 1024L; List events = Lists.newArrayList(); events.add(new DataEventDependencyInfo(timestamp, taId)); events.add(new DataEventDependencyInfo(timestamp, taId)); TaskAttemptFinishedEvent event = new TaskAttemptFinishedEvent( - TezTaskAttemptID.getInstance(TezTaskID.getInstance(TezVertexID.getInstance( - TezDAGID.getInstance(ApplicationId.newInstance(0, 1), 1), 111), 1), 1), - "vertex1", 10001l, 1000434444l, TaskAttemptState.SUCCEEDED, null, null, - null, new TezCounters(), events, null, 0, null, 0, null, null, null, null, null); + TezTaskAttemptID.getInstance(TezTaskID.getInstance(TezVertexID.getInstance( + TezDAGID.getInstance(ApplicationId.newInstance(0, 1), 1), 111), 1), 1), + "vertex1", 10001l, 1000434444l, TaskAttemptState.SUCCEEDED, null, null, + null, new TezCounters(), events, null, 0, null, 0, null, null, null, null, null); String eventStr = event.toString(); - String[] items = new String[] {"taskFailureType", "errorEnum", "diagnostics", "containerId", + String[] items = new String[]{"taskFailureType", "errorEnum", "diagnostics", "containerId", "nodeId", "nodeHttpAddress"}; for (String item : items) { assertFalse(eventStr.contains(item)); @@ -681,7 +679,7 @@ private void testContainerStoppedEvent() throws Exception { ContainerId.newInstance(ApplicationAttemptId.newInstance( ApplicationId.newInstance(0, 1), 1), 1001), 100034566, ContainerExitStatus.SUCCESS, ApplicationAttemptId.newInstance( - ApplicationId.newInstance(0, 1), 1)); + ApplicationId.newInstance(0, 1), 1)); ContainerStoppedEvent deserializedEvent = (ContainerStoppedEvent) testProtoConversion(event); Assert.assertEquals(event.getContainerId(), @@ -766,7 +764,6 @@ private void testVertexGroupCommitFinishedEvent() throws Exception { } } - @Test//(timeout = 5000) public void testDefaultProtoConversion() throws Exception { for (HistoryEventType eventType : HistoryEventType.values()) { @@ -842,9 +839,9 @@ public void testDefaultProtoConversion() throws Exception { break; default: throw new Exception("Unhandled Event type in Unit tests: " + eventType); - } } } + } private void testDAGRecoveredEvent() throws TezException { DAGRecoveredEvent dagRecoveredEvent = new DAGRecoveredEvent( @@ -859,7 +856,5 @@ private void testDAGRecoveredEvent() throws TezException { } catch (IOException e) { Assert.fail("Proto conversion should have failed with Unsupported Exception"); } - } - } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java b/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java index 081e47fbf2..3b646a728e 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -221,9 +221,9 @@ public void testConvertVertexReconfigureDoneEvent() throws JSONException { ApplicationId.newInstance(1l, 1), 1), 1); Map edgeMgrs = new HashMap(); - + edgeMgrs.put("a", EdgeProperty.create(EdgeManagerPluginDescriptor.create("a.class") - .setHistoryText("text"), DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, + .setHistoryText("text"), DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"))); VertexConfigurationDoneEvent event = new VertexConfigurationDoneEvent(vId, 0L, 1, null, edgeMgrs, null, true); @@ -253,8 +253,5 @@ public void testConvertVertexReconfigureDoneEvent() throws JSONException { updatedEdgeMgr.getString(DAGUtils.DATA_MOVEMENT_TYPE_KEY)); Assert.assertEquals("In", updatedEdgeMgr.getString(DAGUtils.EDGE_DESTINATION_CLASS_KEY)); Assert.assertEquals("a.class", updatedEdgeMgr.getString(DAGUtils.EDGE_MANAGER_CLASS_KEY)); - } - - } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/recovery/TestRecoveryService.java b/tez-dag/src/test/java/org/apache/tez/dag/history/recovery/TestRecoveryService.java index 5d1fff9121..7074378b19 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/history/recovery/TestRecoveryService.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/history/recovery/TestRecoveryService.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -135,7 +135,7 @@ public void testDrainEvents() throws Exception { setup(false, null); recoveryService.start(); int randEventCount = new Random().nextInt(100) + 100; - for (int i=0; i< randEventCount; ++i) { + for (int i = 0; i < randEventCount; ++i) { recoveryService.handle(new DAGHistoryEvent(dagId, new TaskStartedEvent(tezTaskId, "v1", 0L, 0L))); } @@ -148,7 +148,7 @@ public void testMultipleDAGFinishedEvent() throws Exception { setup(false, null); recoveryService.start(); int randEventCount = new Random().nextInt(100) + 100; - for (int i=0; i< randEventCount; ++i) { + for (int i = 0; i < randEventCount; ++i) { recoveryService.handle(new DAGHistoryEvent(dagId, new TaskStartedEvent(tezTaskId, "v1", 0L, 0L))); } @@ -203,12 +203,12 @@ public void testRecoveryPathExisted() throws Exception { recoveryService.stop(); } - @Test(timeout=5000) + @Test(timeout = 5000) public void testRecoveryFlushOnMaxEvents() throws Exception { - setup(true, new String[][] { + setup(true, new String[][]{ {TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, "10"}, {TezConfiguration.DAG_RECOVERY_FLUSH_INTERVAL_SECS, "-1"} - }); + }); recoveryService.start(); // Send 1 event less, wait for drain @@ -228,11 +228,11 @@ public void testRecoveryFlushOnMaxEvents() throws Exception { recoveryService.stop(); } - @Test(timeout=10000) + @Test(timeout = 10000) public void testRecoveryFlushOnTimeoutEvents() throws Exception { - setup(true, new String[][] { - {TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, "-1"}, - {TezConfiguration.DAG_RECOVERY_FLUSH_INTERVAL_SECS, "5"} + setup(true, new String[][]{ + {TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, "-1"}, + {TezConfiguration.DAG_RECOVERY_FLUSH_INTERVAL_SECS, "5"} }); recoveryService.start(); @@ -256,11 +256,11 @@ public void testRecoveryFlushOnTimeoutEvents() throws Exception { recoveryService.stop(); } - @Test(timeout=10000) + @Test(timeout = 10000) public void testRecoveryFlush() throws Exception { - setup(true, new String[][] { - {TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, "10"}, - {TezConfiguration.DAG_RECOVERY_FLUSH_INTERVAL_SECS, "5"} + setup(true, new String[][]{ + {TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, "10"}, + {TezConfiguration.DAG_RECOVERY_FLUSH_INTERVAL_SECS, "5"} }); recoveryService.start(); @@ -294,11 +294,11 @@ public void testRecoveryFlush() throws Exception { recoveryService.stop(); } - @Test(timeout=50000) + @Test(timeout = 50000) public void testRecoveryFlushOnStop() throws Exception { - setup(true, new String[][] { - {TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, "-1"}, - {TezConfiguration.DAG_RECOVERY_FLUSH_INTERVAL_SECS, "-1"} + setup(true, new String[][]{ + {TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, "-1"}, + {TezConfiguration.DAG_RECOVERY_FLUSH_INTERVAL_SECS, "-1"} }); recoveryService.start(); @@ -322,11 +322,11 @@ public void testRecoveryFlushOnStop() throws Exception { verify(dagFos, times(1)).hflush(); } - @Test(timeout=5000) + @Test(timeout = 5000) public void testRecoveryFlushOnSummaryEvent() throws Exception { - setup(true, new String[][] { - {TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, "-1"}, - {TezConfiguration.DAG_RECOVERY_FLUSH_INTERVAL_SECS, "-1"} + setup(true, new String[][]{ + {TezConfiguration.DAG_RECOVERY_MAX_UNFLUSHED_EVENTS, "-1"}, + {TezConfiguration.DAG_RECOVERY_FLUSH_INTERVAL_SECS, "-1"} }); recoveryService.start(); @@ -403,7 +403,7 @@ public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize) thro @Override public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, - int bufferSize, short replication, long blockSize, Progressable progress) + int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { return delegate.create(f, permission, overwrite, bufferSize, replication, blockSize, progress); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java b/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java index 4d4577ac77..42108cf3de 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -103,7 +103,7 @@ private DAGPlan createDAG() { @SuppressWarnings("unchecked") public void testConvertDAGPlanToATSMap() throws IOException, JSONException { DAGPlan dagPlan = createDAG(); - Map idNameMap = new HashMap(); + Map idNameMap = new HashMap(); ApplicationId appId = ApplicationId.newInstance(1, 1); TezDAGID dagId = TezDAGID.getInstance(appId, 1); TezVertexID vId1 = TezVertexID.getInstance(dagId, 1); @@ -118,7 +118,7 @@ public void testConvertDAGPlanToATSMap() throws IOException, JSONException { Assert.assertEquals("testDag", atsMap.get(DAGUtils.DAG_NAME_KEY)); Assert.assertTrue(atsMap.containsKey(DAGUtils.DAG_INFO_KEY)); Assert.assertTrue(atsMap.containsKey(DAGUtils.DAG_CONTEXT_KEY)); - Map contextMap = (Map)atsMap.get(DAGUtils.DAG_CONTEXT_KEY); + Map contextMap = (Map) atsMap.get(DAGUtils.DAG_CONTEXT_KEY); Assert.assertEquals("context1", contextMap.get(ATSConstants.CONTEXT)); Assert.assertEquals("callerId1", contextMap.get(ATSConstants.CALLER_ID)); Assert.assertEquals("callerType1", contextMap.get(ATSConstants.CALLER_TYPE)); @@ -143,7 +143,7 @@ public void testConvertDAGPlanToATSMap() throws IOException, JSONException { for (Object o : ((Collection) atsMap.get(DAGUtils.VERTICES_KEY))) { Map v = (Map) o; Assert.assertTrue(v.containsKey(DAGUtils.VERTEX_NAME_KEY)); - String vName = (String)v.get(DAGUtils.VERTEX_NAME_KEY); + String vName = (String) v.get(DAGUtils.VERTEX_NAME_KEY); Assert.assertTrue(v.containsKey(DAGUtils.PROCESSOR_CLASS_KEY)); Assert.assertTrue(v.containsKey(DAGUtils.USER_PAYLOAD_AS_TEXT)); @@ -215,5 +215,4 @@ public void testConvertDAGPlanToATSMap() throws IOException, JSONException { Assert.assertTrue(e.containsKey(DAGUtils.VERTEX_GROUP_EDGE_MERGED_INPUTS_KEY)); } } - } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/utils/TestSimple2LevelVersionComparator.java b/tez-dag/src/test/java/org/apache/tez/dag/utils/TestSimple2LevelVersionComparator.java index f383c37105..09dd3833a5 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/utils/TestSimple2LevelVersionComparator.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/utils/TestSimple2LevelVersionComparator.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -61,6 +61,4 @@ public void testInequalityChecks() { Assert.assertEquals(-1, comparator.compare("1.1", "")); Assert.assertEquals(-1, comparator.compare("", "1.1")); } - - } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/utils/TestTaskSpecificLaunchCmdOption.java b/tez-dag/src/test/java/org/apache/tez/dag/utils/TestTaskSpecificLaunchCmdOption.java index 5685a9740d..3651ff1cec 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/utils/TestTaskSpecificLaunchCmdOption.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/utils/TestTaskSpecificLaunchCmdOption.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -41,7 +41,6 @@ private TaskSpecificLaunchCmdOption getOptions(Configuration conf, String tasks, return new TaskSpecificLaunchCmdOption(conf); } - @Test(timeout = 5000) public void testTaskSpecificJavaOptions() { Random rnd = new Random(); @@ -273,7 +272,6 @@ public void testConfigOptions() { assertTrue(optionStr.equals("dir=/tmp/v/4")); } - @Test(timeout = 5000) public void testTaskSpecificLogOptions() { Configuration conf = new Configuration(false); @@ -298,7 +296,7 @@ public void testTaskSpecificLogOptions() { assertEquals(1, options.getTaskSpecificLogParams().length); } - @Test (timeout=5000) + @Test(timeout = 5000) public void testTaskSpecificLogOptionsWithCommandOptions() { Configuration conf = new Configuration(false); conf.set(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST, "v1[0,2,5]"); diff --git a/tez-dag/src/test/java/org/apache/tez/test/ControlledScheduledExecutorService.java b/tez-dag/src/test/java/org/apache/tez/test/ControlledScheduledExecutorService.java index f6da15b81a..c07155c970 100644 --- a/tez-dag/src/test/java/org/apache/tez/test/ControlledScheduledExecutorService.java +++ b/tez-dag/src/test/java/org/apache/tez/test/ControlledScheduledExecutorService.java @@ -39,7 +39,9 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -/** A scheduled executor service with timing that can be controlled for unit tests. */ +/** + * A scheduled executor service with timing that can be controlled for unit tests. + */ public class ControlledScheduledExecutorService implements ScheduledExecutorService, MockClockListener { private final MockClock clock; private final PriorityQueue> queue = new PriorityQueue<>(); @@ -152,12 +154,12 @@ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedE @Override public void onTimeUpdated(long newTime) { - ScheduledFutureTask task = queue.peek(); - while (task != null && task.timestamp <= newTime) { - task = queue.poll(); - runTask(task); - task = queue.peek(); - } + ScheduledFutureTask task = queue.peek(); + while (task != null && task.timestamp <= newTime) { + task = queue.poll(); + runTask(task); + task = queue.peek(); + } } private long now() { diff --git a/tez-dag/src/test/java/org/apache/tez/test/EdgeManagerForTest.java b/tez-dag/src/test/java/org/apache/tez/test/EdgeManagerForTest.java index 47d133b315..1a1ccc67eb 100644 --- a/tez-dag/src/test/java/org/apache/tez/test/EdgeManagerForTest.java +++ b/tez-dag/src/test/java/org/apache/tez/test/EdgeManagerForTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -61,7 +61,7 @@ public int getNumSourceTaskPhysicalOutputs(int sourceTaskIndex) { @Override public void routeDataMovementEventToDestination(DataMovementEvent event, int sourceTaskIndex, - int sourceOutputIndex, Map> destinationTaskAndInputIndices) { + int sourceOutputIndex, Map> destinationTaskAndInputIndices) { } @Override @@ -76,12 +76,12 @@ public int routeInputErrorEventToSource(InputReadErrorEvent event, int destinati @Override public void routeInputSourceTaskFailedEventToDestination(int sourceTaskIndex, - Map> destinationTaskAndInputIndices) { + Map> destinationTaskAndInputIndices) { } @Override public EventRouteMetadata routeDataMovementEventToDestination(int sourceTaskIndex, - int sourceOutputIndex, int destinationTaskIndex) throws Exception { + int sourceOutputIndex, int destinationTaskIndex) throws Exception { return null; } @@ -107,7 +107,6 @@ public int routeInputErrorEventToSource(int destinationTaskIndex, int destinatio throws Exception { return 0; } - - // End of overridden methods -} \ No newline at end of file + // End of overridden methods +} diff --git a/tez-dag/src/test/java/org/apache/tez/test/GraceShuffleVertexManagerForTest.java b/tez-dag/src/test/java/org/apache/tez/test/GraceShuffleVertexManagerForTest.java index ff89ef83eb..896b124768 100644 --- a/tez-dag/src/test/java/org/apache/tez/test/GraceShuffleVertexManagerForTest.java +++ b/tez-dag/src/test/java/org/apache/tez/test/GraceShuffleVertexManagerForTest.java @@ -39,7 +39,7 @@ * A shuffle vertex manager that will set the vertex's parallelism upon * completion of its single grandparent simulating a very simplified * version of PigGraceShuffleVertexManager for testing purposes. - * + *

    * This manager plugin should only be used for vertices that have a single * grandparent. */ diff --git a/tez-dag/src/test/java/org/apache/tez/test/VertexManagerPluginForTest.java b/tez-dag/src/test/java/org/apache/tez/test/VertexManagerPluginForTest.java index 1cdaeca5c4..35cbea807b 100644 --- a/tez-dag/src/test/java/org/apache/tez/test/VertexManagerPluginForTest.java +++ b/tez-dag/src/test/java/org/apache/tez/test/VertexManagerPluginForTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -38,29 +38,29 @@ public class VertexManagerPluginForTest extends VertexManagerPlugin { VertexManagerPluginForTestConfig pluginConfig = new VertexManagerPluginForTestConfig(); - + public static class VertexManagerPluginForTestConfig { Configuration conf = new Configuration(false); static final String RECONFIGURE_ON_START = "reconfigureOnStart"; static final String NUM_TASKS = "numTasks"; - + public void setReconfigureOnStart(boolean value) { conf.setBoolean(RECONFIGURE_ON_START, value); } - + public void setNumTasks(int value) { conf.setInt(NUM_TASKS, value); } - + boolean getReconfigureOnStart() { return conf.getBoolean(RECONFIGURE_ON_START, false); } - + int getNumTasks() { return conf.getInt(NUM_TASKS, 1); } - + public ByteBuffer getPayload() { ByteArrayOutputStream b = new ByteArrayOutputStream(); DataOutputStream out = new DataOutputStream(b); @@ -71,7 +71,7 @@ public ByteBuffer getPayload() { } return ByteBuffer.wrap(b.toByteArray()); } - + void initialize(ByteBuffer buff) { ByteBuffer copy = ByteBuffer.allocate(buff.capacity()); copy.put(buff); @@ -85,10 +85,10 @@ void initialize(ByteBuffer buff) { } } } + public VertexManagerPluginForTest(VertexManagerPluginContext context) { super(context); } - @Override public void initialize() { @@ -113,5 +113,5 @@ public void onVertexManagerEventReceived(VertexManagerEvent vmEvent) {} @Override public void onRootVertexInitialized(String inputName, - InputDescriptor inputDescriptor, List events) {} + InputDescriptor inputDescriptor, List events) {} } diff --git a/tez-examples/src/main/java/org/apache/tez/examples/CartesianProduct.java b/tez-examples/src/main/java/org/apache/tez/examples/CartesianProduct.java index a925137f97..064f7eba38 100644 --- a/tez-examples/src/main/java/org/apache/tez/examples/CartesianProduct.java +++ b/tez-examples/src/main/java/org/apache/tez/examples/CartesianProduct.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -85,7 +85,7 @@ public class CartesianProduct extends TezExampleBase { private static final String UNPARTITIONED = "-unpartitioned"; private static final Logger LOG = LoggerFactory.getLogger(CartesianProduct.class); private static final int numPartition = 2; - private static final String[] cpSources = new String[] {VERTEX1, VERTEX2}; + private static final String[] cpSources = new String[]{VERTEX1, VERTEX2}; public static class TokenProcessor extends SimpleProcessor { public TokenProcessor(ProcessorContext context) { @@ -148,20 +148,20 @@ public int getPartition(Object key, Object value, int numPartitions) { private DAG createDAG(TezConfiguration tezConf, String inputPath1, String inputPath2, String inputPath3, String outputPath, boolean isPartitioned) - throws IOException { + throws IOException { Vertex v1 = Vertex.create(VERTEX1, ProcessorDescriptor.create(TokenProcessor.class.getName())); // turn off groupSplit so that each input file incurs one task v1.addDataSource(INPUT, - MRInput.createConfigBuilder(new Configuration(tezConf), TextInputFormat.class, inputPath1) - .groupSplits(false).build()); + MRInput.createConfigBuilder(new Configuration(tezConf), TextInputFormat.class, inputPath1) + .groupSplits(false).build()); Vertex v2 = Vertex.create(VERTEX2, ProcessorDescriptor.create(TokenProcessor.class.getName())); v2.addDataSource(INPUT, - MRInput.createConfigBuilder(new Configuration(tezConf), TextInputFormat.class, inputPath2) - .groupSplits(false).build()); + MRInput.createConfigBuilder(new Configuration(tezConf), TextInputFormat.class, inputPath2) + .groupSplits(false).build()); Vertex v3 = Vertex.create(VERTEX3, ProcessorDescriptor.create(TokenProcessor.class.getName())); v3.addDataSource(INPUT, - MRInput.createConfigBuilder(new Configuration(tezConf), TextInputFormat.class, inputPath3) - .groupSplits(false).build()); + MRInput.createConfigBuilder(new Configuration(tezConf), TextInputFormat.class, inputPath3) + .groupSplits(false).build()); CartesianProductConfig cartesianProductConfig; if (isPartitioned) { Map vertexPartitionMap = new HashMap<>(); @@ -175,54 +175,54 @@ private DAG createDAG(TezConfiguration tezConf, String inputPath1, String inputP UserPayload userPayload = cartesianProductConfig.toUserPayload(tezConf); Vertex v4 = Vertex.create(VERTEX4, ProcessorDescriptor.create(JoinProcessor.class.getName())); v4.addDataSink(OUTPUT, - MROutput.createConfigBuilder(new Configuration(tezConf), TextOutputFormat.class, outputPath) - .build()); + MROutput.createConfigBuilder(new Configuration(tezConf), TextOutputFormat.class, outputPath) + .build()); v4.setVertexManagerPlugin( - VertexManagerPluginDescriptor.create(CartesianProductVertexManager.class.getName()) - .setUserPayload(userPayload)); + VertexManagerPluginDescriptor.create(CartesianProductVertexManager.class.getName()) + .setUserPayload(userPayload)); EdgeManagerPluginDescriptor cpEdgeManager = - EdgeManagerPluginDescriptor.create(CartesianProductEdgeManager.class.getName()); + EdgeManagerPluginDescriptor.create(CartesianProductEdgeManager.class.getName()); cpEdgeManager.setUserPayload(userPayload); EdgeProperty cpEdgeProperty; if (isPartitioned) { UnorderedPartitionedKVEdgeConfig cpEdgeConf = - UnorderedPartitionedKVEdgeConfig.newBuilder(Text.class.getName(), - IntWritable.class.getName(), CustomPartitioner.class.getName()).build(); + UnorderedPartitionedKVEdgeConfig.newBuilder(Text.class.getName(), + IntWritable.class.getName(), CustomPartitioner.class.getName()).build(); cpEdgeProperty = cpEdgeConf.createDefaultCustomEdgeProperty(cpEdgeManager); } else { UnorderedKVEdgeConfig edgeConf = - UnorderedKVEdgeConfig.newBuilder(Text.class.getName(), IntWritable.class.getName()).build(); + UnorderedKVEdgeConfig.newBuilder(Text.class.getName(), IntWritable.class.getName()).build(); cpEdgeProperty = edgeConf.createDefaultCustomEdgeProperty(cpEdgeManager); } EdgeProperty broadcastEdgeProperty; UnorderedKVEdgeConfig broadcastEdgeConf = - UnorderedKVEdgeConfig.newBuilder(Text.class.getName(), IntWritable.class.getName()).build(); + UnorderedKVEdgeConfig.newBuilder(Text.class.getName(), IntWritable.class.getName()).build(); broadcastEdgeProperty = broadcastEdgeConf.createDefaultBroadcastEdgeProperty(); return DAG.create("CartesianProduct") - .addVertex(v1).addVertex(v2).addVertex(v3).addVertex(v4) - .addEdge(Edge.create(v1, v4, cpEdgeProperty)) - .addEdge(Edge.create(v2, v4, cpEdgeProperty)) - .addEdge(Edge.create(v3, v4, broadcastEdgeProperty)); + .addVertex(v1).addVertex(v2).addVertex(v3).addVertex(v4) + .addEdge(Edge.create(v1, v4, cpEdgeProperty)) + .addEdge(Edge.create(v2, v4, cpEdgeProperty)) + .addEdge(Edge.create(v3, v4, broadcastEdgeProperty)); } @Override protected void printUsage() { - System.err.println("Usage: args: ["+PARTITIONED + "|" + UNPARTITIONED - + " "); + System.err.println("Usage: args: [" + PARTITIONED + "|" + UNPARTITIONED + + " "); } @Override protected int validateArgs(String[] otherArgs) { return (otherArgs.length != 5 || (!otherArgs[0].equals(PARTITIONED) - && !otherArgs[0].equals(UNPARTITIONED))) ? -1 : 0; + && !otherArgs[0].equals(UNPARTITIONED))) ? -1 : 0; } @Override protected int runJob(String[] args, TezConfiguration tezConf, - TezClient tezClient) throws Exception { + TezClient tezClient) throws Exception { DAG dag = createDAG(tezConf, args[1], args[2], args[3], args[4], args[0].equals(PARTITIONED)); return runDag(dag, isCountersLog(), LOG); @@ -233,4 +233,3 @@ public static void main(String[] args) throws Exception { System.exit(res); } } - diff --git a/tez-examples/src/main/java/org/apache/tez/examples/ExampleDriver.java b/tez-examples/src/main/java/org/apache/tez/examples/ExampleDriver.java index 074f77fa19..ba28ded143 100644 --- a/tez-examples/src/main/java/org/apache/tez/examples/ExampleDriver.java +++ b/tez-examples/src/main/java/org/apache/tez/examples/ExampleDriver.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -40,7 +40,7 @@ public class ExampleDriver { private static final DecimalFormat formatter = new DecimalFormat("###.##%"); - public static void main(String argv[]){ + public static void main(String argv[]) { int exitCode = -1; ProgramDriver pgd = new ProgramDriver(); try { @@ -61,12 +61,10 @@ public static void main(String argv[]){ pgd.addClass("cartesianproduct", CartesianProduct.class, "Cartesian product of two datasets"); exitCode = pgd.run(argv); - } catch(Throwable e){ + } catch (Throwable e) { e.printStackTrace(); } System.exit(exitCode); } - } - diff --git a/tez-examples/src/main/java/org/apache/tez/examples/HashJoinExample.java b/tez-examples/src/main/java/org/apache/tez/examples/HashJoinExample.java index 6eaa03f9d7..e29072cf2c 100644 --- a/tez-examples/src/main/java/org/apache/tez/examples/HashJoinExample.java +++ b/tez-examples/src/main/java/org/apache/tez/examples/HashJoinExample.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -97,7 +97,7 @@ protected void printUsage() { @Override protected int runJob(String[] args, TezConfiguration tezConf, - TezClient tezClient) throws Exception { + TezClient tezClient) throws Exception { boolean doBroadcast = args.length == 5 && args[4].equals(broadcastOption) ? true : false; @@ -140,7 +140,7 @@ protected int validateArgs(String[] otherArgs) { } private DAG createDag(TezConfiguration tezConf, Path streamPath, - Path hashPath, Path outPath, int numPartitions, boolean doBroadcast) + Path hashPath, Path outPath, int numPartitions, boolean doBroadcast) throws IOException { DAG dag = DAG.create("HashJoinExample" + (doBroadcast ? "-WithBroadcast" : "")); @@ -152,7 +152,7 @@ private DAG createDag(TezConfiguration tezConf, Path streamPath, */ Vertex hashFileVertex = Vertex.create(hashSide, - ProcessorDescriptor.create(ForwardingProcessor.class.getName())) + ProcessorDescriptor.create(ForwardingProcessor.class.getName())) .addDataSource( inputFile, MRInput @@ -169,7 +169,7 @@ private DAG createDag(TezConfiguration tezConf, Path streamPath, */ Vertex streamFileVertex = Vertex.create(streamingSide, - ProcessorDescriptor.create(ForwardingProcessor.class.getName())) + ProcessorDescriptor.create(ForwardingProcessor.class.getName())) .addDataSource( inputFile, MRInput diff --git a/tez-examples/src/main/java/org/apache/tez/examples/JoinDataGen.java b/tez-examples/src/main/java/org/apache/tez/examples/JoinDataGen.java index d096f32158..f13f760fac 100644 --- a/tez-examples/src/main/java/org/apache/tez/examples/JoinDataGen.java +++ b/tez-examples/src/main/java/org/apache/tez/examples/JoinDataGen.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -72,7 +72,7 @@ protected void printUsage() { @Override protected int runJob(String[] args, TezConfiguration tezConf, - TezClient tezClient) throws Exception { + TezClient tezClient) throws Exception { LOG.info("Running JoinDataGen"); String outDir1 = args[0]; @@ -130,7 +130,7 @@ protected int validateArgs(String[] otherArgs) { } private DAG createDag(TezConfiguration tezConf, Path largeOutPath, Path smallOutPath, - Path expectedOutputPath, int numTasks, long largeOutSize, long smallOutSize) + Path expectedOutputPath, int numTasks, long largeOutSize, long smallOutSize) throws IOException { long largeOutSizePerTask = largeOutSize / numTasks; @@ -142,13 +142,13 @@ private DAG createDag(TezConfiguration tezConf, Path largeOutPath, Path smallOut GenDataProcessor.class.getName()).setUserPayload( UserPayload.create(ByteBuffer.wrap(GenDataProcessor.createConfiguration(largeOutSizePerTask, smallOutSizePerTask)))), numTasks); - genDataVertex.addDataSink(STREAM_OUTPUT_NAME, + genDataVertex.addDataSink(STREAM_OUTPUT_NAME, MROutput.createConfigBuilder(new Configuration(tezConf), TextOutputFormat.class, largeOutPath.toUri().toString()).build()); - genDataVertex.addDataSink(HASH_OUTPUT_NAME, + genDataVertex.addDataSink(HASH_OUTPUT_NAME, MROutput.createConfigBuilder(new Configuration(tezConf), TextOutputFormat.class, smallOutPath.toUri().toString()).build()); - genDataVertex.addDataSink(EXPECTED_OUTPUT_NAME, + genDataVertex.addDataSink(EXPECTED_OUTPUT_NAME, MROutput.createConfigBuilder(new Configuration(tezConf), TextOutputFormat.class, expectedOutputPath.toUri().toString()).build()); @@ -276,7 +276,6 @@ private String createNonOverlaptring(int size, long count) { sb.append("_").append(getContext().getTaskIndex()).append("_").append(count); return sb.toString(); } - } private int checkOutputDirectory(Configuration conf, Path path) throws IOException { @@ -288,5 +287,4 @@ private int checkOutputDirectory(Configuration conf, Path path) throws IOExcepti } return 0; } - } diff --git a/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java b/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java index a5d3ebd711..155c7a5bd4 100644 --- a/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java +++ b/tez-examples/src/main/java/org/apache/tez/examples/JoinValidate.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -76,7 +76,7 @@ protected void printUsage() { @Override protected int runJob(String[] args, TezConfiguration tezConf, - TezClient tezClient) throws Exception { + TezClient tezClient) throws Exception { LOG.info("Running JoinValidate"); @@ -124,7 +124,6 @@ protected int runJob(String[] args, TezConfiguration tezConf, } } } - } @Override @@ -158,7 +157,7 @@ DAG createDag(TezConfiguration tezConf, Path lhs, Path rhs, int numPartitions) MRInput .createConfigBuilder(new Configuration(tezConf), TextInputFormat.class, lhs.toUri().toString()).groupSplits(!isDisableSplitGrouping()) - .generateSplitsInAM(!isGenerateSplitInClient()).build()); + .generateSplitsInAM(!isGenerateSplitInClient()).build()); setVertexExecutionContext(lhsVertex, getLhsExecutionContext()); Vertex rhsVertex = Vertex.create(RHS_INPUT_NAME, ProcessorDescriptor.create( @@ -166,7 +165,7 @@ DAG createDag(TezConfiguration tezConf, Path lhs, Path rhs, int numPartitions) MRInput .createConfigBuilder(new Configuration(tezConf), TextInputFormat.class, rhs.toUri().toString()).groupSplits(!isDisableSplitGrouping()) - .generateSplitsInAM(!isGenerateSplitInClient()).build()); + .generateSplitsInAM(!isGenerateSplitInClient()).build()); setVertexExecutionContext(rhsVertex, getRhsExecutionContext()); Vertex joinValidateVertex = Vertex.create("joinvalidate", ProcessorDescriptor.create( @@ -259,6 +258,4 @@ public void run() throws Exception { } } } - - } diff --git a/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java b/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java index 12fd943afa..2deb7887e8 100644 --- a/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java +++ b/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -55,7 +55,7 @@ * The example extends WordCount by sorting the words by their count. */ public class OrderedWordCount extends TezExampleBase { - + private static String INPUT = WordCount.INPUT; private static String OUTPUT = WordCount.OUTPUT; private static String TOKENIZER = WordCount.TOKENIZER; @@ -64,8 +64,8 @@ public class OrderedWordCount extends TezExampleBase { private static final Logger LOG = LoggerFactory.getLogger(OrderedWordCount.class); /* - * SumProcessor similar to WordCount except that it writes the count as key and the - * word as value. This is because we can and ordered partitioned key value edge to group the + * SumProcessor similar to WordCount except that it writes the count as key and the + * word as value. This is because we can and ordered partitioned key value edge to group the * words with the same count (as key) and order the counts. */ public static class SumProcessor extends SimpleProcessor { @@ -95,7 +95,7 @@ public void run() throws Exception { } } } - + /** * No-op sorter processor. It does not need to apply any logic since the ordered partitioned edge * ensures that we get the data sorted and grouped by the the sum key. @@ -121,13 +121,13 @@ public void run() throws Exception { // deriving from SimpleMRProcessor takes care of committing the output } } - + public static DAG createDAG(TezConfiguration tezConf, String inputPath, String outputPath, - int numPartitions, boolean disableSplitGrouping, boolean isGenerateSplitInClient, String dagName) throws IOException { + int numPartitions, boolean disableSplitGrouping, boolean isGenerateSplitInClient, String dagName) throws IOException { DataSourceDescriptor dataSource = MRInput.createConfigBuilder(new Configuration(tezConf), - TextInputFormat.class, inputPath).groupSplits(!disableSplitGrouping) - .generateSplitsInAM(!isGenerateSplitInClient).build(); + TextInputFormat.class, inputPath).groupSplits(!disableSplitGrouping) + .generateSplitsInAM(!isGenerateSplitInClient).build(); DataSinkDescriptor dataSink = MROutput.createConfigBuilder(new Configuration(tezConf), TextOutputFormat.class, outputPath).build(); @@ -149,7 +149,7 @@ public static DAG createDAG(TezConfiguration tezConf, String inputPath, String o // via an output edge. Vertex summationVertex = Vertex.create(SUMMATION, ProcessorDescriptor.create( SumProcessor.class.getName()), numPartitions); - + // Use IntWritable key and Text value to bring all words with the same count in the same // partition. The data will be ordered by count and words grouped by count. The // setFromConfiguration call is optional and allows overriding the config options with @@ -167,7 +167,7 @@ public static DAG createDAG(TezConfiguration tezConf, String inputPath, String o sorterVertex.addDataSink(OUTPUT, dataSink); // No need to add jar containing this class as assumed to be part of the tez jars. - + DAG dag = DAG.create(dagName); dag.addVertex(tokenizerVertex) .addVertex(summationVertex) @@ -177,7 +177,7 @@ public static DAG createDAG(TezConfiguration tezConf, String inputPath, String o summationEdgeConf.createDefaultEdgeProperty())) .addEdge( Edge.create(summationVertex, sorterVertex, sorterEdgeConf.createDefaultEdgeProperty())); - return dag; + return dag; } @Override @@ -185,7 +185,6 @@ protected void printUsage() { System.err.println("Usage: " + " orderedwordcount in out [numPartitions]"); } - @Override protected int validateArgs(String[] otherArgs) { if (otherArgs.length < 2 || otherArgs.length > 3) { @@ -196,7 +195,7 @@ protected int validateArgs(String[] otherArgs) { @Override protected int runJob(String[] args, TezConfiguration tezConf, - TezClient tezClient) throws Exception { + TezClient tezClient) throws Exception { DAG dag = createDAG(tezConf, args[0], args[1], args.length == 3 ? Integer.parseInt(args[2]) : 1, isDisableSplitGrouping(), isGenerateSplitInClient(), "OrderedWordCount"); diff --git a/tez-examples/src/main/java/org/apache/tez/examples/SimpleSessionExample.java b/tez-examples/src/main/java/org/apache/tez/examples/SimpleSessionExample.java index d555f471a4..634d191804 100644 --- a/tez-examples/src/main/java/org/apache/tez/examples/SimpleSessionExample.java +++ b/tez-examples/src/main/java/org/apache/tez/examples/SimpleSessionExample.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -56,7 +56,7 @@ protected int validateArgs(String[] otherArgs) { @Override protected int runJob(String[] args, TezConfiguration tezConf, - TezClient tezClient) throws Exception { + TezClient tezClient) throws Exception { System.out.println("Running SimpleSessionExample"); String[] inputPaths = args[0].split(","); String[] outputPaths = args[1].split(","); @@ -90,7 +90,7 @@ protected int runJob(String[] args, TezConfiguration tezConf, isDisableSplitGrouping(), isGenerateSplitInClient(), ("DAG-Iteration-" + i)); // the names of the DAGs must be unique in a session LOG.info("Running dag number " + i); - if(runDag(dag, isCountersLog(), LOG) != 0) { + if (runDag(dag, isCountersLog(), LOG) != 0) { return -1; } } diff --git a/tez-examples/src/main/java/org/apache/tez/examples/SortMergeJoinExample.java b/tez-examples/src/main/java/org/apache/tez/examples/SortMergeJoinExample.java index 02156d684e..d8e02d7767 100644 --- a/tez-examples/src/main/java/org/apache/tez/examples/SortMergeJoinExample.java +++ b/tez-examples/src/main/java/org/apache/tez/examples/SortMergeJoinExample.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -93,7 +93,7 @@ protected void printUsage() { @Override protected int runJob(String[] args, TezConfiguration tezConf, - TezClient tezClient) throws Exception { + TezClient tezClient) throws Exception { String inputDir1 = args[0]; String inputDir2 = args[1]; @@ -133,7 +133,7 @@ protected int validateArgs(String[] otherArgs) { * v1 v2
    *  \ /
    *   v3
    - * + * * @param tezConf * @param inputPath1 * @param inputPath2 @@ -143,7 +143,7 @@ protected int validateArgs(String[] otherArgs) { * @throws IOException */ private DAG createDag(TezConfiguration tezConf, Path inputPath1, - Path inputPath2, Path outPath, int numPartitions) throws IOException { + Path inputPath2, Path outPath, int numPartitions) throws IOException { DAG dag = DAG.create("SortMergeJoinExample"); /** @@ -153,7 +153,7 @@ private DAG createDag(TezConfiguration tezConf, Path inputPath1, */ Vertex inputVertex1 = Vertex.create("input1", - ProcessorDescriptor.create(ForwardingProcessor.class.getName())) + ProcessorDescriptor.create(ForwardingProcessor.class.getName())) .addDataSource( inputFile, MRInput @@ -169,7 +169,7 @@ private DAG createDag(TezConfiguration tezConf, Path inputPath1, */ Vertex inputVertex2 = Vertex.create("input2", - ProcessorDescriptor.create(ForwardingProcessor.class.getName())) + ProcessorDescriptor.create(ForwardingProcessor.class.getName())) .addDataSource( inputFile, MRInput @@ -269,14 +269,14 @@ public void run() throws Exception { /** * Join 2 sorted inputs both from {@link KeyValuesReader} and write output * using {@link KeyValueWriter} - * + * * @param inputReader1 * @param inputReader2 * @param writer * @throws IOException */ private void join(KeyValuesReader inputReader1, - KeyValuesReader inputReader2, KeyValueWriter writer) throws IOException { + KeyValuesReader inputReader2, KeyValueWriter writer) throws IOException { while (inputReader1.next() && inputReader2.next()) { Text value1 = (Text) inputReader1.getCurrentKey(); diff --git a/tez-examples/src/main/java/org/apache/tez/examples/TezExampleBase.java b/tez-examples/src/main/java/org/apache/tez/examples/TezExampleBase.java index cb521055e4..b70ba3492f 100644 --- a/tez-examples/src/main/java/org/apache/tez/examples/TezExampleBase.java +++ b/tez-examples/src/main/java/org/apache/tez/examples/TezExampleBase.java @@ -66,7 +66,6 @@ public abstract class TezExampleBase extends Configured implements Tool { protected static final String LEAVE_AM_RUNNING = "leaveAmRunning"; protected static final String RECONNECT_APP_ID = "reconnectAppId"; - private boolean disableSplitGrouping = false; private boolean isLocalMode = false; private boolean isCountersLog = false; @@ -76,7 +75,7 @@ public abstract class TezExampleBase extends Configured implements Tool { private HadoopShim hadoopShim; protected boolean isCountersLog() { - return isCountersLog; + return isCountersLog; } protected boolean isDisableSplitGrouping() { @@ -90,8 +89,8 @@ protected boolean isGenerateSplitInClient() { private Options getExtraOptions() { Options options = new Options(); options.addOption(LOCAL_MODE, false, "run it as local mode"); - options.addOption(DISABLE_SPLIT_GROUPING, false , "disable split grouping"); - options.addOption(COUNTER_LOG, false , "print counter log"); + options.addOption(DISABLE_SPLIT_GROUPING, false, "disable split grouping"); + options.addOption(COUNTER_LOG, false, "print counter log"); options.addOption(GENERATE_SPLIT_IN_CLIENT, false, "whether generate split in client"); options.addOption(LEAVE_AM_RUNNING, false, "whether client should stop session"); options.addOption(RECONNECT_APP_ID, true, "appId for client reconnect"); @@ -119,7 +118,7 @@ public final int run(String[] args) throws Exception { leaveAmRunning = true; } if (optionParser.getCommandLine().hasOption(RECONNECT_APP_ID)) { - reconnectAppId = optionParser.getCommandLine().getOptionValue(RECONNECT_APP_ID); + reconnectAppId = optionParser.getCommandLine().getOptionValue(RECONNECT_APP_ID); } hadoopShim = new HadoopShimsLoader(conf).getHadoopShim(); @@ -137,7 +136,7 @@ public final int run(String[] args) throws Exception { * provided configuration. If TezClient is specified, local mode option can not been * specified in arguments, it takes no effect. * @return Zero indicates success, non-zero indicates failure - * @throws Exception + * @throws Exception */ public int run(TezConfiguration conf, String[] args, @Nullable TezClient tezClient) throws Exception { @@ -252,7 +251,7 @@ private int _execute(String[] otherArgs, TezConfiguration tezConf, TezClient tez private TezClient createTezClient(TezConfiguration tezConf) throws IOException, TezException { TezClient tezClient = TezClient.create("TezExampleApplication", tezConf); - if(reconnectAppId != null) { + if (reconnectAppId != null) { ApplicationId appId = TezClient.appIdfromString(reconnectAppId); tezClient.getClient(appId); } else { @@ -311,7 +310,7 @@ protected void printExtraOptionsUsage(PrintStream ps) { */ protected abstract int runJob(String[] args, TezConfiguration tezConf, TezClient tezClient) throws Exception; - + @Private @VisibleForTesting public ApplicationId getAppId() { diff --git a/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java b/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java index c58c3bad75..5307e1d73e 100644 --- a/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java +++ b/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -98,16 +98,15 @@ public void run() throws Exception { } } } - } /* * Example code to write a processor that commits final output to a data sink - * The SumProcessor aggregates the sum of individual word counts generated by + * The SumProcessor aggregates the sum of individual word counts generated by * the TokenProcessor. * The SumProcessor is connected to a DataSink. In this case, its an Output that * writes the data via an OutputFormat to a data sink (typically HDFS). Thats why - * it derives from SimpleMRProcessor that takes care of handling the necessary + * it derives from SimpleMRProcessor that takes care of handling the necessary * output commit operations that makes the final output available for consumers. */ public static class SumProcessor extends SimpleMRProcessor { @@ -138,13 +137,13 @@ public void run() throws Exception { } private DAG createDAG(TezConfiguration tezConf, String inputPath, String outputPath, - int numPartitions) throws IOException { + int numPartitions) throws IOException { // Create the descriptor that describes the input data to Tez. Using MRInput to read text // data from the given input path. The TextInputFormat is used to read the text data. DataSourceDescriptor dataSource = MRInput.createConfigBuilder(new Configuration(tezConf), - TextInputFormat.class, inputPath).groupSplits(!isDisableSplitGrouping()) - .generateSplitsInAM(!isGenerateSplitInClient()).build(); + TextInputFormat.class, inputPath).groupSplits(!isDisableSplitGrouping()) + .generateSplitsInAM(!isGenerateSplitInClient()).build(); // Create a descriptor that describes the output data to Tez. Using MROoutput to write text // data to the given output path. The TextOutputFormat is used to write the text data. @@ -180,19 +179,19 @@ private DAG createDAG(TezConfiguration tezConf, String inputPath, String outputP // to distribute the sum processing. In this case, its been made configurable via the // numPartitions parameter. Vertex summationVertex = Vertex.create(SUMMATION, - ProcessorDescriptor.create(SumProcessor.class.getName()), numPartitions) + ProcessorDescriptor.create(SumProcessor.class.getName()), numPartitions) .addDataSink(OUTPUT, dataSink); // No need to add jar containing this class as assumed to be part of the Tez jars. Otherwise // we would have to add the jars for this code as local files to the vertices. - + // Create DAG and add the vertices. Connect the producer and consumer vertices via the edge DAG dag = DAG.create("WordCount"); dag.addVertex(tokenizerVertex) .addVertex(summationVertex) .addEdge( Edge.create(tokenizerVertex, summationVertex, edgeConf.createDefaultEdgeProperty())); - return dag; + return dag; } @Override @@ -210,7 +209,7 @@ protected int validateArgs(String[] otherArgs) { @Override protected int runJob(String[] args, TezConfiguration tezConf, - TezClient tezClient) throws Exception { + TezClient tezClient) throws Exception { DAG dag = createDAG(tezConf, args[0], args[1], args.length == 3 ? Integer.parseInt(args[2]) : 1); LOG.info("Running WordCount"); diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/TezTestServiceCommunicator.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/TezTestServiceCommunicator.java index 713a3d3fcd..c0c28746ad 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/TezTestServiceCommunicator.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/TezTestServiceCommunicator.java @@ -14,7 +14,6 @@ package org.apache.tez.dag.app; - import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -56,7 +55,6 @@ public void serviceStop() { executor.shutdownNow(); } - public void runContainer(RunContainerRequestProto request, String host, int port, final ExecuteRequestCallback callback) { ListenableFuture future = executor.submit(new RunContainerCallable(request, host, port)); @@ -71,7 +69,6 @@ public void onFailure(Throwable t) { callback.indicateError(t); } }, GuavaShim.directExecutor()); - } public void submitWork(SubmitWorkRequestProto request, String host, int port, @@ -88,10 +85,8 @@ public void onFailure(Throwable t) { callback.indicateError(t); } }, GuavaShim.directExecutor()); - } - private class RunContainerCallable implements Callable { final String hostname; @@ -100,7 +95,7 @@ private class RunContainerCallable implements Callable { void setResponse(T response); + void indicateError(Throwable t); } diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java index 110d918edf..e6fc204e2b 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceContainerLauncher.java @@ -51,7 +51,6 @@ public class TezTestServiceContainerLauncher extends ContainerLauncher { private final ApplicationAttemptId appAttemptId; private final Configuration conf; - // Configuration passed in here to set up final parameters public TezTestServiceContainerLauncher(ContainerLauncherContext containerLauncherContext) { super(containerLauncherContext); @@ -145,6 +144,4 @@ private RunContainerRequestProto constructRunContainerRequest(ContainerLaunchReq void sendContainerLaunchFailedMsg(ContainerId containerId, Throwable t) { getContext().containerLaunchFailed(containerId, t == null ? "" : t.getMessage()); } - - } diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceNoOpContainerLauncher.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceNoOpContainerLauncher.java index d2657364da..8f20dadf94 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceNoOpContainerLauncher.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/launcher/TezTestServiceNoOpContainerLauncher.java @@ -25,7 +25,6 @@ public class TezTestServiceNoOpContainerLauncher extends ContainerLauncher { static final Logger LOG = LoggerFactory.getLogger(TezTestServiceNoOpContainerLauncher.class); - public TezTestServiceNoOpContainerLauncher(ContainerLauncherContext containerLauncherContext) { super(containerLauncherContext); } diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java index d7d0f01126..fdc7be70bd 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/rm/TezTestServiceTaskSchedulerService.java @@ -42,7 +42,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class TezTestServiceTaskSchedulerService extends TaskScheduler { private static final Logger @@ -67,7 +66,6 @@ public class TezTestServiceTaskSchedulerService extends TaskScheduler { // Per Executor Thread private final Resource resourcePerContainer; - // Not registering with the RM. Assuming the main TezScheduler will always run (except local mode), // and take care of YARN registration. public TezTestServiceTaskSchedulerService(TaskSchedulerContext taskSchedulerContext) { @@ -122,7 +120,6 @@ public TezTestServiceTaskSchedulerService(TaskSchedulerContext taskSchedulerCont ", executorsPerInstance=" + executorsPerInstance + ", resourcePerContainerInferred=" + resourcePerContainer + ", hosts=" + serviceHosts.toString()); - } @Override @@ -169,7 +166,6 @@ public void allocateTask(Object task, Resource capability, String[] hosts, Strin getContext().taskAllocated(task, clientCookie, container); } - @Override public void allocateTask(Object task, Resource capability, ContainerId containerId, Priority priority, Object containerSignature, Object clientCookie) { diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java index 377217b0cf..5f8761218f 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/dag/app/taskcomm/TezTestServiceTaskCommunicatorImpl.java @@ -43,7 +43,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class TezTestServiceTaskCommunicatorImpl extends TezTaskCommunicatorImpl { private static final Logger @@ -91,7 +90,6 @@ public void shutdown() { this.communicator.stop(); } - @Override public void registerRunningContainer(ContainerId containerId, String hostname, int port) { super.registerRunningContainer(containerId, hostname, port); @@ -107,7 +105,7 @@ public void registerRunningTaskAttempt(final ContainerId containerId, final Task Map additionalResources, Credentials credentials, boolean credentialsChanged, - int priority) { + int priority) { super.registerRunningTaskAttempt(containerId, taskSpec, additionalResources, credentials, credentialsChanged, priority); SubmitWorkRequestProto requestProto = null; diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/examples/JoinValidateConfigured.java b/tez-ext-service-tests/src/test/java/org/apache/tez/examples/JoinValidateConfigured.java index 64b9063922..384dbae55b 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/examples/JoinValidateConfigured.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/examples/JoinValidateConfigured.java @@ -14,7 +14,6 @@ package org.apache.tez.examples; - import java.io.IOException; import org.apache.hadoop.fs.Path; diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/ContainerRunner.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/ContainerRunner.java index 28c228654f..7e7e4fdc06 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/service/ContainerRunner.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/ContainerRunner.java @@ -24,5 +24,6 @@ public interface ContainerRunner { void queueContainer(RunContainerRequestProto request) throws TezException; + void submitWork(SubmitWorkRequestProto request) throws TezException; } diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/MiniTezTestServiceCluster.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/MiniTezTestServiceCluster.java index 3a6935f52b..39d264b0f5 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/service/MiniTezTestServiceCluster.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/MiniTezTestServiceCluster.java @@ -100,7 +100,7 @@ private MiniTezTestServiceCluster(String clusterName, int numExecutorsPerService // Setup Local Dirs localDirs = new String[numLocalDirs]; - for (int i = 0 ; i < numLocalDirs ; i++) { + for (int i = 0; i < numLocalDirs; i++) { File f = new File(testWorkDir, "localDir"); f.mkdirs(); LOG.info("Created localDir: " + f.getAbsolutePath()); @@ -112,7 +112,6 @@ private MiniTezTestServiceCluster(String clusterName, int numExecutorsPerService public void serviceInit(Configuration conf) { tezTestService = new TezTestService(conf, numExecutorsPerService, availableMemory, localDirs); tezTestService.init(conf); - } @Override @@ -141,6 +140,7 @@ public void serviceStop() { /** * return the address at which the service is listening + * * @return host:port */ public InetSocketAddress getServiceAddress() { @@ -162,5 +162,4 @@ public Configuration getClusterSpecificConfiguration() { public int getNumSubmissions() { return tezTestService.getNumSubmissions(); } - -} \ No newline at end of file +} diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/TezTestServiceConfConstants.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/TezTestServiceConfConstants.java index bf4a5bdae1..9a1db6e4f1 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/service/TezTestServiceConfConstants.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/TezTestServiceConfConstants.java @@ -18,24 +18,34 @@ public class TezTestServiceConfConstants { private static final String TEZ_TEST_SERVICE_PREFIX = "tez.test.service."; - /** Number of executors per instance - used by the scheduler */ + /** + * Number of executors per instance - used by the scheduler + */ public static final String TEZ_TEST_SERVICE_NUM_EXECUTORS_PER_INSTANCE = TEZ_TEST_SERVICE_PREFIX + "num.executors.per-instance"; - /** Memory available per instance - used by the scheduler */ + /** + * Memory available per instance - used by the scheduler + */ public static final String TEZ_TEST_SERVICE_MEMORY_PER_INSTANCE_MB = TEZ_TEST_SERVICE_PREFIX + "memory.per.instance.mb"; - /** CPUs available per instance - used by the scheduler */ + /** + * CPUs available per instance - used by the scheduler + */ public static final String TEZ_TEST_SERVICE_VCPUS_PER_INSTANCE = TEZ_TEST_SERVICE_PREFIX + "vcpus.per.instance"; - - /** Hosts on which the service is running. Currently assuming a single port for all instances */ + /** + * Hosts on which the service is running. Currently assuming a single port for all instances + */ public static final String TEZ_TEST_SERVICE_HOSTS = TEZ_TEST_SERVICE_PREFIX + "hosts"; - /** Port on which the Service(s) listen. Current a single port for all instances */ + /** + * Port on which the Service(s) listen. Current a single port for all instances + */ public static final String TEZ_TEST_SERVICE_RPC_PORT = TEZ_TEST_SERVICE_PREFIX + "rpc.port"; - /** Number of threads to use in the AM to communicate with the external service */ + /** + * Number of threads to use in the AM to communicate with the external service + */ public static final String TEZ_TEST_SERVICE_AM_COMMUNICATOR_NUM_THREADS = TEZ_TEST_SERVICE_PREFIX + "communicator.num.threads"; public static final int TEZ_TEST_SERVICE_AM_COMMUNICATOR_NUM_THREADS_DEFAULT = 2; - } diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java index 7864e1c852..c9279f9d27 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/ContainerRunnerImpl.java @@ -96,9 +96,6 @@ public class ContainerRunnerImpl extends AbstractService implements ContainerRun private final TezExecutors sharedExecutor; // TODO Support for removing queued containers, interrupting / killing specific containers - when preemption is supported - - - public ContainerRunnerImpl(int numExecutors, String[] localDirsBase, AtomicReference localAddress, long totalMemoryAvailableBytes, @@ -113,10 +110,9 @@ public ContainerRunnerImpl(int numExecutors, String[] localDirsBase, new ThreadFactoryBuilder().setNameFormat("ContainerExecutor %d").build()); this.executorService = MoreExecutors.listeningDecorator(raw); - // 80% of memory considered for accounted buffers. Rest for objects. // TODO Tune this based on the available size. - this.memoryPerExecutor = (long)(totalMemoryAvailableBytes * 0.8 / (float) numExecutors); + this.memoryPerExecutor = (long) (totalMemoryAvailableBytes * 0.8 / (float) numExecutors); LOG.info("ContainerRunnerImpl config: " + "memoryPerExecutorDerived=" + memoryPerExecutor + @@ -159,6 +155,7 @@ private static String createAppSpecificLocalDir(String baseDir, String applicati /** * Submit a container which is ready for running. * The regular pull mechanism will be used to fetch work from the AM + * * @param request * @throws TezException */ @@ -184,7 +181,6 @@ public void queueContainer(RunContainerRequestProto request) throws TezException } LOG.info("Dirs for {} are {}", request.getContainerIdString(), Arrays.toString(localDirs)); - // Setup workingDir. This is otherwise setup as Environment.PWD // Used for re-localization, to add the user specified configuration (conf_pb_binary_stream) String workingDir = localDirs[0]; @@ -205,7 +201,6 @@ public void queueContainer(RunContainerRequestProto request) throws TezException LOG.info("Registering request with the ShuffleHandler for containerId {}", request.getContainerIdString()); ShuffleHandler.get().registerApplication(request.getApplicationIdString(), jobToken, request.getUser()); - ContainerRunnerCallable callable = new ContainerRunnerCallable(request, new Configuration(getConfig()), new ExecutionContextImpl(localAddress.get().getHostName()), env, localDirs, workingDir, credentials, memoryPerExecutor); @@ -274,7 +269,6 @@ public void submitWork(SubmitWorkRequestProto request) throws TezException { Futures.addCallback(future, new TaskRunnerCallback(request, callable), GuavaShim.directExecutor()); } - static class ContainerRunnerCallable implements Callable { private final RunContainerRequestProto request; @@ -289,7 +283,6 @@ static class ContainerRunnerCallable implements Callable envMap, String[] localDirs, String workingDir, Credentials credentials, @@ -303,7 +296,6 @@ static class ContainerRunnerCallable implements Callable { private final RunContainerRequestProto request; @@ -394,11 +385,10 @@ static class TaskRunnerCallable implements Callable { private TezTaskUmbilicalProtocol umbilical; private final TezExecutors sharedExecutor; - TaskRunnerCallable(SubmitWorkRequestProto request, Configuration conf, - ExecutionContext executionContext, Map envMap, - String[] localDirs, String workingDir, Credentials credentials, - long memoryAvailable, TezExecutors sharedExecutor) { + ExecutionContext executionContext, Map envMap, + String[] localDirs, String workingDir, Credentials credentials, + long memoryAvailable, TezExecutors sharedExecutor) { this.request = request; this.conf = conf; this.executionContext = executionContext; @@ -502,14 +492,13 @@ public void shutdown() { } } - final class TaskRunnerCallback implements FutureCallback { private final SubmitWorkRequestProto request; private final TaskRunnerCallable taskRunnerCallable; TaskRunnerCallback(SubmitWorkRequestProto request, - TaskRunnerCallable containerRunnerCallable) { + TaskRunnerCallable containerRunnerCallable) { this.request = request; this.taskRunnerCallable = containerRunnerCallable; } @@ -549,7 +538,6 @@ public void onFailure(Throwable t) { } } - private void checkAndThrowExceptionForTests(SubmitWorkRequestProto request) throws TezException { if (!request.getTaskSpec().getDagName().equals(DAG_NAME_INSTRUMENTED_FAILURES)) { return; diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestService.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestService.java index db14991c79..234ecc34fa 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestService.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestService.java @@ -46,7 +46,6 @@ public class TezTestService extends AbstractService implements ContainerRunner { private final AtomicInteger numSubmissions = new AtomicInteger(0); - private final AtomicReference address = new AtomicReference(); private final TezExecutors sharedExecutor; @@ -114,8 +113,6 @@ public int getShufflePort() { return ShuffleHandler.get().getPort(); } - - @Override public void queueContainer(RunContainerRequestProto request) throws TezException { numSubmissions.incrementAndGet(); diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolClientImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolClientImpl.java index 10d2952a46..d4f2d4a697 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolClientImpl.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolClientImpl.java @@ -28,14 +28,12 @@ import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerRequestProto; import org.apache.tez.test.service.rpc.TezTestServiceProtocolProtos.RunContainerResponseProto; - public class TezTestServiceProtocolClientImpl implements TezTestServiceProtocolBlockingPB { private final Configuration conf; private final InetSocketAddress serverAddr; TezTestServiceProtocolBlockingPB proxy; - public TezTestServiceProtocolClientImpl(Configuration conf, String hostname, int port) { this.conf = conf; this.serverAddr = NetUtils.createSocketAddr(hostname, port); @@ -63,7 +61,6 @@ public TezTestServiceProtocolProtos.SubmitWorkResponseProto submitWork(RpcContro } } - public TezTestServiceProtocolBlockingPB getProxy() throws IOException { if (proxy == null) { proxy = createProxy(); @@ -79,4 +76,4 @@ public TezTestServiceProtocolBlockingPB createProxy() throws IOException { .getProxy(TezTestServiceProtocolBlockingPB.class, 0, serverAddr, conf); return p; } -} \ No newline at end of file +} diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java index b5d3f83e84..b7b6866d92 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/service/impl/TezTestServiceProtocolServerImpl.java @@ -48,7 +48,6 @@ public class TezTestServiceProtocolServerImpl extends AbstractService private RPC.Server server; private final AtomicReference bindAddress; - public TezTestServiceProtocolServerImpl(ContainerRunner containerRunner, AtomicReference address) { super(TezTestServiceProtocolServerImpl.class.getSimpleName()); @@ -81,7 +80,6 @@ public SubmitWorkResponseProto submitWork(RpcController controller, SubmitWorkRe return SubmitWorkResponseProto.getDefaultInstance(); } - @Override public void serviceStart() { Configuration conf = getConfig(); diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/IndexCache.java b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/IndexCache.java index 51224cd8cf..934ee9758d 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/IndexCache.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/IndexCache.java @@ -34,8 +34,8 @@ class IndexCache { private AtomicInteger totalMemoryUsed = new AtomicInteger(); private static final Logger LOG = LoggerFactory.getLogger(IndexCache.class); - private final ConcurrentHashMap cache = - new ConcurrentHashMap(); + private final ConcurrentHashMap cache = + new ConcurrentHashMap(); private final LinkedBlockingQueue queue = new LinkedBlockingQueue(); @@ -59,16 +59,17 @@ private void initLocalFs() { /** * This method gets the index information for the given mapId and reduce. * It reads the index file into cache if it is not already present. + * * @param mapId * @param reduce - * @param fileName The file to read the index information from if it is not - * already present in the cache + * @param fileName The file to read the index information from if it is not + * already present in the cache * @param expectedIndexOwner The expected owner of the index file * @return The Index Information * @throws IOException */ public TezIndexRecord getIndexInformation(String mapId, int reduce, - Path fileName, String expectedIndexOwner) + Path fileName, String expectedIndexOwner) throws IOException { IndexInformation info = cache.get(mapId); @@ -76,7 +77,7 @@ public TezIndexRecord getIndexInformation(String mapId, int reduce, if (info == null) { info = readIndexFileToCache(fileName, mapId, expectedIndexOwner); } else { - synchronized(info) { + synchronized (info) { while (isUnderConstruction(info)) { try { info.wait(); @@ -98,7 +99,7 @@ public TezIndexRecord getIndexInformation(String mapId, int reduce, } private boolean isUnderConstruction(IndexInformation info) { - synchronized(info) { + synchronized (info) { return (null == info.mapSpillRecord); } } @@ -110,7 +111,7 @@ private IndexInformation readIndexFileToCache(Path indexFileName, IndexInformation info; IndexInformation newInd = new IndexInformation(); if ((info = cache.putIfAbsent(mapId, newInd)) != null) { - synchronized(info) { + synchronized (info) { while (isUnderConstruction(info)) { try { info.wait(); @@ -122,7 +123,7 @@ private IndexInformation readIndexFileToCache(Path indexFileName, LOG.debug("IndexCache HIT: MapId " + mapId + " found"); return info; } - LOG.debug("IndexCache MISS: MapId " + mapId + " not found") ; + LOG.debug("IndexCache MISS: MapId " + mapId + " not found"); TezSpillRecord tmp = null; try { tmp = new TezSpillRecord(indexFileName, fs, expectedIndexOwner); @@ -146,10 +147,11 @@ private IndexInformation readIndexFileToCache(Path indexFileName, /** * This method removes the map from the cache if index information for this - * map is loaded(size>0), index information entry in cache will not be - * removed if it is in the loading phrase(size=0), this prevents corruption - * of totalMemoryUsed. It should be called when a map output on this tracker + * map is loaded(size>0), index information entry in cache will not be + * removed if it is in the loading phrase(size=0), this prevents corruption + * of totalMemoryUsed. It should be called when a map output on this tracker * is discarded. + * * @param mapId The taskID of this map. */ public void removeMap(String mapId) { @@ -171,6 +173,7 @@ public void removeMap(String mapId) { /** * This method checks if cache and totolMemoryUsed is consistent. * It is only used for unit test. + * * @return True if cache and totolMemoryUsed is consistent */ boolean checkTotalMemoryUsed() { diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/ShuffleHandler.java b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/ShuffleHandler.java index 43f24ba654..461e537e26 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/ShuffleHandler.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/shufflehandler/ShuffleHandler.java @@ -129,7 +129,7 @@ public class ShuffleHandler { */ private final int maxShuffleConnections; - private Map userRsrc; + private Map userRsrc; private JobTokenSecretManager secretManager; public static final String SHUFFLE_PORT_CONFIG_KEY = "tez.shuffle.port"; @@ -153,11 +153,11 @@ public class ShuffleHandler { public static final String MAX_SHUFFLE_CONNECTIONS = "mapreduce.shuffle.max.connections"; public static final int DEFAULT_MAX_SHUFFLE_CONNECTIONS = 0; // 0 implies no limit - + public static final String MAX_SHUFFLE_THREADS = "mapreduce.shuffle.max.threads"; // 0 implies Netty default of 2 * number of available processors public static final int DEFAULT_MAX_SHUFFLE_THREADS = 0; - + final boolean connectionKeepAliveEnabled; final int connectionKeepAliveTimeOut; final int mapOutputMetaInfoCacheSize; @@ -165,7 +165,6 @@ public class ShuffleHandler { private static final AtomicBoolean initing = new AtomicBoolean(false); private static ShuffleHandler INSTANCE; - public ShuffleHandler(Configuration conf) { this.conf = conf; @@ -205,11 +204,10 @@ public Thread newThread(Runnable r) { Math.max(1, conf.getInt(SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE, DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE)); - userRsrc = new ConcurrentHashMap(); + userRsrc = new ConcurrentHashMap(); secretManager = new JobTokenSecretManager(); } - public void start() throws Exception { ServerBootstrap bootstrap = new ServerBootstrap() .channel(NioServerSocketChannel.class) @@ -219,7 +217,7 @@ public void start() throws Exception { port = conf.getInt(SHUFFLE_PORT_CONFIG_KEY, DEFAULT_SHUFFLE_PORT); Channel ch = bootstrap.bind().sync().channel(); accepted.add(ch); - port = ((InetSocketAddress)ch.localAddress()).getPort(); + port = ((InetSocketAddress) ch.localAddress()).getPort(); conf.set(SHUFFLE_PORT_CONFIG_KEY, Integer.toString(port)); SHUFFLE.setPort(port); LOG.info("TezShuffleHandler" + " listening on port " + port); @@ -237,15 +235,15 @@ private void initPipeline(ServerBootstrap bootstrap, Configuration conf) throws ChannelInitializer channelInitializer = new ChannelInitializer() { @Override - public void initChannel(NioSocketChannel ch) throws Exception { - ChannelPipeline pipeline = ch.pipeline(); - pipeline.addLast("decoder", new HttpRequestDecoder()); - pipeline.addLast("aggregator", new HttpObjectAggregator(1 << 16)); - pipeline.addLast("encoder", new HttpResponseEncoder()); - pipeline.addLast("chunking", new ChunkedWriteHandler()); - pipeline.addLast("shuffle", SHUFFLE); - } - }; + public void initChannel(NioSocketChannel ch) throws Exception { + ChannelPipeline pipeline = ch.pipeline(); + pipeline.addLast("decoder", new HttpRequestDecoder()); + pipeline.addLast("aggregator", new HttpObjectAggregator(1 << 16)); + pipeline.addLast("encoder", new HttpResponseEncoder()); + pipeline.addLast("chunking", new ChunkedWriteHandler()); + pipeline.addLast("shuffle", SHUFFLE); + } + }; bootstrap.childHandler(channelInitializer); } @@ -294,7 +292,7 @@ protected Shuffle getShuffle(Configuration conf) { } private void addJobToken(String appIdString, String user, - Token jobToken) { + Token jobToken) { String jobIdString = appIdString.replace("application", "job"); userRsrc.put(jobIdString, user); secretManager.addTokenForJob(jobIdString, jobToken); @@ -302,7 +300,7 @@ private void addJobToken(String appIdString, String user, } private void recordJobShuffleInfo(String appIdString, String user, - Token jobToken) { + Token jobToken) { addJobToken(appIdString, user, jobToken); } @@ -317,7 +315,7 @@ class Shuffle extends ChannelInboundHandlerAdapter { private final Configuration conf; private final IndexCache indexCache; private final LocalDirAllocator lDirAlloc = - new LocalDirAllocator(SHUFFLE_HANDLER_LOCAL_DIRS); + new LocalDirAllocator(SHUFFLE_HANDLER_LOCAL_DIRS); private int port; public Shuffle(Configuration conf) { @@ -325,7 +323,7 @@ public Shuffle(Configuration conf) { indexCache = new IndexCache(conf); this.port = conf.getInt(SHUFFLE_PORT_CONFIG_KEY, DEFAULT_SHUFFLE_PORT); } - + public void setPort(int port) { this.port = port; } @@ -347,7 +345,7 @@ public void channelActive(ChannelHandlerContext ctx) if ((maxShuffleConnections > 0) && (accepted.size() >= maxShuffleConnections)) { LOG.info(String.format("Current number of shuffle connections (%d) is " + - "greater than or equal to the max allowed shuffle connections (%d)", + "greater than or equal to the max allowed shuffle connections (%d)", accepted.size(), maxShuffleConnections)); ctx.channel().close(); return; @@ -361,14 +359,14 @@ public void channelRead(ChannelHandlerContext ctx, Object message) throws Exception { HttpRequest request = (HttpRequest) message; if (request.getMethod() != GET) { - sendError(ctx, METHOD_NOT_ALLOWED); - return; + sendError(ctx, METHOD_NOT_ALLOWED); + return; } // Check whether the shuffle version is compatible if (!ShuffleHeader.DEFAULT_HTTP_HEADER_NAME.equals( request.headers().get(ShuffleHeader.HTTP_HEADER_NAME)) || !ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION.equals( - request.headers().get(ShuffleHeader.HTTP_HEADER_VERSION))) { + request.headers().get(ShuffleHeader.HTTP_HEADER_VERSION))) { sendError(ctx, "Incompatible shuffle request version", BAD_REQUEST); } final Map> q = new QueryStringDecoder(request.getUri()).parameters(); @@ -378,7 +376,7 @@ public void channelRead(ChannelHandlerContext ctx, Object message) keepAliveParam = Boolean.valueOf(keepAliveList.get(0)); if (LOG.isDebugEnabled()) { LOG.debug("KeepAliveParam : " + keepAliveList - + " : " + keepAliveParam); + + " : " + keepAliveParam); } } final List mapIds = splitMaps(q.get("map")); @@ -440,12 +438,12 @@ public void channelRead(ChannelHandlerContext ctx, Object message) try { populateHeaders(mapIds, outputBasePathStr, user, reduceId, request, - response, keepAliveParam, mapOutputInfoMap); - } catch(IOException e) { + response, keepAliveParam, mapOutputInfoMap); + } catch (IOException e) { ch.writeAndFlush(response); LOG.error("Shuffle error in populating headers :", e); String errorMessage = getErrorMessage(e); - sendError(ctx,errorMessage , INTERNAL_SERVER_ERROR); + sendError(ctx, errorMessage, INTERNAL_SERVER_ERROR); return; } ch.writeAndFlush(response); @@ -459,7 +457,7 @@ public void channelRead(ChannelHandlerContext ctx, Object message) } lastMap = sendMapOutput(ctx, ch, user, mapId, - reduceId, info); + reduceId, info); if (null == lastMap) { sendError(ctx, NOT_FOUND); return; @@ -467,7 +465,7 @@ public void channelRead(ChannelHandlerContext ctx, Object message) } catch (IOException e) { LOG.error("Shuffle error :", e); String errorMessage = getErrorMessage(e); - sendError(ctx,errorMessage , INTERNAL_SERVER_ERROR); + sendError(ctx, errorMessage, INTERNAL_SERVER_ERROR); return; } } @@ -499,7 +497,7 @@ private String getBaseLocation(String jobIdString, String user) { } protected MapOutputInfo getMapOutputInfo(String base, String mapId, - int reduce, String user) throws IOException { + int reduce, String user) throws IOException { // Index file Path indexFileName = lDirAlloc.getLocalPathToRead(base + "/file.out.index", conf); @@ -516,8 +514,8 @@ protected MapOutputInfo getMapOutputInfo(String base, String mapId, } protected void populateHeaders(List mapIds, String outputBaseStr, - String user, int reduce, HttpRequest request, HttpResponse response, - boolean keepAliveParam, Map mapOutputInfoMap) + String user, int reduce, HttpRequest request, HttpResponse response, + boolean keepAliveParam, Map mapOutputInfoMap) throws IOException { long contentLength = 0; @@ -546,13 +544,13 @@ protected void populateHeaders(List mapIds, String outputBaseStr, } protected void setResponseHeaders(HttpResponse response, - boolean keepAliveParam, long contentLength) { + boolean keepAliveParam, long contentLength) { if (!connectionKeepAliveEnabled && !keepAliveParam) { LOG.info("Setting connection close header..."); response.headers().set(HttpHeaders.Names.CONNECTION, CONNECTION_CLOSE); } else { response.headers().set(HttpHeaders.Names.CONTENT_LENGTH, - String.valueOf(contentLength)); + String.valueOf(contentLength)); response.headers().set(HttpHeaders.Names.CONNECTION, HttpHeaders.Values.KEEP_ALIVE); response.headers().set(HttpHeaders.Values.KEEP_ALIVE, "timeout=" + connectionKeepAliveTimeOut); @@ -571,7 +569,7 @@ class MapOutputInfo { } protected void verifyRequest(String appid, ChannelHandlerContext ctx, - HttpRequest request, HttpResponse response, URL requestUri) + HttpRequest request, HttpResponse response, URL requestUri) throws IOException { SecretKey tokenSecret = secretManager.retrieveTokenSecret(appid); if (null == tokenSecret) { @@ -582,7 +580,7 @@ protected void verifyRequest(String appid, ChannelHandlerContext ctx, String enc_str = SecureShuffleUtils.buildMsgFrom(requestUri); // hash from the fetcher String urlHashStr = - request.headers().get(SecureShuffleUtils.HTTP_HEADER_URL_HASH); + request.headers().get(SecureShuffleUtils.HTTP_HEADER_URL_HASH); if (urlHashStr == null) { LOG.info("Missing header hash for " + appid); throw new IOException("fetcher cannot be authenticated"); @@ -590,14 +588,14 @@ protected void verifyRequest(String appid, ChannelHandlerContext ctx, if (LOG.isDebugEnabled()) { int len = urlHashStr.length(); LOG.debug("verifying request. enc_str=" + enc_str + "; hash=..." + - urlHashStr.substring(len-len/2, len-1)); + urlHashStr.substring(len - len / 2, len - 1)); } // verify - throws exception SecureShuffleUtils.verifyReply(urlHashStr, enc_str, tokenSecret); // verification passed - encode the reply String reply = - SecureShuffleUtils.generateHash(urlHashStr.getBytes(Charsets.UTF_8), - tokenSecret); + SecureShuffleUtils.generateHash(urlHashStr.getBytes(Charsets.UTF_8), + tokenSecret); response.headers().set(SecureShuffleUtils.HTTP_HEADER_REPLY_URL_HASH, reply); // Put shuffle version into http header response.headers().set(ShuffleHeader.HTTP_HEADER_NAME, @@ -607,16 +605,16 @@ protected void verifyRequest(String appid, ChannelHandlerContext ctx, if (LOG.isDebugEnabled()) { int len = reply.length(); LOG.debug("Fetcher request verfied. enc_str=" + enc_str + ";reply=" + - reply.substring(len-len/2, len-1)); + reply.substring(len - len / 2, len - 1)); } } protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, Channel ch, - String user, String mapId, int reduce, MapOutputInfo mapOutputInfo) + String user, String mapId, int reduce, MapOutputInfo mapOutputInfo) throws IOException { final TezIndexRecord info = mapOutputInfo.indexRecord; final ShuffleHeader header = - new ShuffleHeader(mapId, info.getPartLength(), info.getRawLength(), reduce); + new ShuffleHeader(mapId, info.getPartLength(), info.getRawLength(), reduce); final DataOutputBuffer dob = new DataOutputBuffer(); header.write(dob); ch.writeAndFlush(wrappedBuffer(dob.getData(), 0, dob.getLength())); @@ -637,12 +635,12 @@ protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, Channel ch, } protected void sendError(ChannelHandlerContext ctx, - HttpResponseStatus status) { + HttpResponseStatus status) { sendError(ctx, "", status); } protected void sendError(ChannelHandlerContext ctx, String message, - HttpResponseStatus status) { + HttpResponseStatus status) { FullHttpResponse response = new DefaultFullHttpResponse(HTTP_1_1, status); response.headers().set(CONTENT_TYPE, "text/plain; charset=UTF-8"); // Put shuffle version into http header diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/ExternalTezServiceTestHelper.java b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/ExternalTezServiceTestHelper.java index c3831223c7..56d710f64b 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/ExternalTezServiceTestHelper.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/ExternalTezServiceTestHelper.java @@ -142,7 +142,6 @@ public void shutdownSharedTezClient() throws IOException, TezException { } } - public void setupHashJoinData(Path srcDataDir, Path dataPath1, Path dataPath2, Path expectedResultPath, Path outputPath) throws Exception { @@ -162,7 +161,6 @@ public void setupHashJoinData(Path srcDataDir, Path dataPath1, Path dataPath2, LOG.info("Completed generating Data - Expected Hash Result and Actual Join Result"); } - public MiniTezCluster getTezCluster() { return tezCluster; } diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExtServicesWithLocalMode.java b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExtServicesWithLocalMode.java index c0bfe76bd4..9057e4178c 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExtServicesWithLocalMode.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExtServicesWithLocalMode.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -54,7 +54,6 @@ public class TestExtServicesWithLocalMode { private static final Logger LOG = LoggerFactory.getLogger(TestExtServicesWithLocalMode.class); - private static final String EXT_PUSH_ENTITY_NAME = "ExtServiceTestPush"; private static String TEST_ROOT_DIR = @@ -111,7 +110,6 @@ public static void tearDown() throws IOException, TezException { localFs.delete(testRootDirPath, true); } - @Test(timeout = 30000) public void test1() throws Exception { @@ -135,7 +133,6 @@ public void test1() throws Exception { ServicePluginsDescriptor servicePluginsDescriptor = ServicePluginsDescriptor.create(true, false, taskSchedulerDescriptors, containerLauncherDescriptors, taskCommunicatorDescriptors); - TezConfiguration tezConf = new TezConfiguration(confForJobs); TezClient tezClient = TezClient.newBuilder("test1", tezConf).setIsSession(true) @@ -143,13 +140,11 @@ public void test1() throws Exception { try { tezClient.start(); - Path dataPath1 = new Path(SRC_DATA_DIR, "inPath1"); Path dataPath2 = new Path(SRC_DATA_DIR, "inPath2"); Path expectedResultPath = new Path(SRC_DATA_DIR, "expectedOutputPath"); - JoinDataGen dataGen = new JoinDataGen(); String[] dataGenArgs = new String[]{ dataPath1.toString(), "1048576", dataPath2.toString(), "524288", @@ -185,7 +180,6 @@ public void test1() throws Exception { } finally { tezClient.stop(); } - } private void runJoinValidate(TezClient tezClient, String name, int extExpectedCount, diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java index c135d7a433..3a18868129 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServices.java @@ -94,10 +94,8 @@ public static void setup() throws Exception { ServicePluginsDescriptor servicePluginsDescriptor = ServicePluginsDescriptor.create(true, true, taskSchedulerDescriptors, containerLauncherDescriptors, taskCommunicatorDescriptors); - extServiceTestHelper.setupSharedTezClient(servicePluginsDescriptor); - // Generate the join data set used for each run. // Can a timeout be enforced here ? Path dataPath1 = new Path(SRC_DATA_DIR, "inPath1"); @@ -111,7 +109,6 @@ public static void tearDown() throws IOException, TezException { extServiceTestHelper.tearDownAll(); } - @Test(timeout = 60000) public void testAllInService() throws Exception { int expectedExternalSubmissions = 4 + 3; //4 for 4 src files, 3 for num reducers. @@ -187,11 +184,9 @@ public void testErrorPropagation() throws TezException, InterruptedException, IO runExceptionSimulation(); } - - private void runExceptionSimulation() throws IOException, TezException, InterruptedException { DAG dag = DAG.create(ContainerRunnerImpl.DAG_NAME_INSTRUMENTED_FAILURES); - Vertex v =Vertex.create("Vertex1", ProcessorDescriptor.create(SleepProcessor.class.getName()), + Vertex v = Vertex.create("Vertex1", ProcessorDescriptor.create(SleepProcessor.class.getName()), 3); v.setExecutionContext(EXECUTION_CONTEXT_EXT_SERVICE_PUSH); dag.addVertex(v); @@ -201,7 +196,6 @@ private void runExceptionSimulation() throws IOException, TezException, Interrup assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState()); assertEquals(1, dagStatus.getDAGProgress().getFailedTaskAttemptCount()); assertEquals(1, dagStatus.getDAGProgress().getRejectedTaskAttemptCount()); - } private void runJoinValidate(String name, int extExpectedCount, VertexExecutionContext lhsContext, diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServicesErrors.java b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServicesErrors.java index ac6ebde428..97d81dd100 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServicesErrors.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExternalTezServicesErrors.java @@ -113,7 +113,6 @@ public class TestExternalTezServicesErrors { private static final Vertex.VertexExecutionContext EXECUTION_CONTEXT_SCHEDULER_REPORT_FATAL = Vertex.VertexExecutionContext.create(EXT_REPORT_FATAL_ERROR_ENTITY_NAME, EXT_PUSH_ENTITY_NAME, EXT_PUSH_ENTITY_NAME); - private static final Vertex.VertexExecutionContext EXECUTION_CONTEXT_DEFAULT = EXECUTION_CONTEXT_EXT_SERVICE_PUSH; private static String TEST_ROOT_DIR = "target" + Path.SEPARATOR + TestExternalTezServicesErrors.class.getName() @@ -222,7 +221,7 @@ public void testTaskSchedulerThrowError() throws Exception { DAGAppMasterEventType.TASK_SCHEDULER_SERVICE_FATAL_ERROR.name())); } - @Test (timeout = 150000) + @Test(timeout = 150000) public void testNonFatalErrors() throws IOException, TezException, InterruptedException { String methodName = "testNonFatalErrors"; TezConfiguration tezClientConf = new TezConfiguration(extServiceTestHelper.getConfForJobs()); @@ -236,11 +235,9 @@ public void testNonFatalErrors() throws IOException, TezException, InterruptedEx tezClient.waitTillReady(); LOG.info("TezSession ready for submission for " + methodName); - runAndVerifyForNonFatalErrors(tezClient, SUFFIX_LAUNCHER, EXECUTION_CONTEXT_LAUNCHER_REPORT_NON_FATAL); runAndVerifyForNonFatalErrors(tezClient, SUFFIX_TASKCOMM, EXECUTION_CONTEXT_TASKCOMM_REPORT_NON_FATAL); runAndVerifyForNonFatalErrors(tezClient, SUFFIX_SCHEDULER, EXECUTION_CONTEXT_SCHEDULER_REPORT_NON_FATAL); - } finally { tezClient.stop(); } @@ -269,7 +266,6 @@ public void testTaskSchedulerReportFatalError() throws Exception { ServicePluginErrorDefaults.INCONSISTENT_STATE.name())); } - private void testFatalError(String methodName, Vertex.VertexExecutionContext lhsExecutionContext, String dagNameSuffix, List expectedDiagMessages) throws @@ -280,7 +276,7 @@ private void testFatalError(String methodName, tezClientConf) .setIsSession(true).setServicePluginDescriptor(servicePluginsDescriptor).build(); - ApplicationId appId= null; + ApplicationId appId = null; try { tezClient.start(); LOG.info("TezSessionStarted for " + methodName); @@ -360,7 +356,6 @@ private boolean checkDiag(String diag, List expected) { return found; } - private void runAndVerifyForNonFatalErrors(TezClient tezClient, String componentName, Vertex.VertexExecutionContext lhsContext) throws TezException, @@ -392,5 +387,4 @@ private void runAndVerifyForNonFatalErrors(TezClient tezClient, String component } assertTrue(foundDiag); } - } diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/util/ProtoConverters.java b/tez-ext-service-tests/src/test/java/org/apache/tez/util/ProtoConverters.java index 25d61d0bf5..8b8ff38b8c 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/util/ProtoConverters.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/util/ProtoConverters.java @@ -115,7 +115,6 @@ public static TaskSpecProto convertTaskSpecToProto(TaskSpec taskSpec) { if (taskSpec.getGroupInputs() != null && !taskSpec.getGroupInputs().isEmpty()) { for (GroupInputSpec groupInputSpec : taskSpec.getGroupInputs()) { builder.addGroupedInputSpecs(convertGroupInputSpecToProto(groupInputSpec)); - } } if (taskSpec.getTaskConf() != null) { @@ -132,7 +131,6 @@ public static TaskSpecProto convertTaskSpecToProto(TaskSpec taskSpec) { return builder.build(); } - public static InputSpec getInputSpecFromProto(IOSpecProto inputSpecProto) { InputDescriptor inputDescriptor = null; if (inputSpecProto.hasIoDescriptor()) { @@ -195,5 +193,4 @@ public static GroupInputSpecProto convertGroupInputSpecToProto(GroupInputSpec gr DagTypeConverters.convertToDAGPlan(groupInputSpec.getMergedInputDescriptor())); return builder.build(); } - } diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/SplitSizeEstimator.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/SplitSizeEstimator.java index a4c0e73d9a..1bf9d7a15d 100644 --- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/SplitSizeEstimator.java +++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/SplitSizeEstimator.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,7 +22,7 @@ import org.apache.hadoop.mapred.InputSplit; -/* the two big differences between the mapred.* and mapreduce.* split classes +/* the two big differences between the mapred.* and mapreduce.* split classes * is that mapreduce throws InterruptedExceptions */ public interface SplitSizeEstimator { long getEstimatedSize(InputSplit split) throws IOException; diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplit.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplit.java index bc58043303..6c711744e6 100644 --- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplit.java +++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplit.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -49,9 +49,9 @@ public class TezGroupedSplit implements InputSplit, Configurable { String rack = null; long length = 0; Configuration conf; - + public TezGroupedSplit() { - + } public TezGroupedSplit(List wrappedSplits, String wrappedInputFormatName, @@ -64,14 +64,15 @@ public TezGroupedSplit(List wrappedSplits, String wrappedInputFormat } public TezGroupedSplit(int numSplits, String wrappedInputFormatName, - String[] locations, String rack) { + String[] locations, String rack) { this.wrappedSplits = new ArrayList(numSplits); this.wrappedInputFormatName = wrappedInputFormatName; this.locations = locations; this.rack = rack; } + public TezGroupedSplit(int numSplits, String wrappedInputFormatName, - String[] locations) { + String[] locations) { this(numSplits, wrappedInputFormatName, locations, null); } @@ -87,7 +88,7 @@ public void addSplit(InputSplit split) { throw new TezUncheckedException(e); } } - + @Override public void write(DataOutput out) throws IOException { if (wrappedSplits == null) { @@ -97,11 +98,11 @@ public void write(DataOutput out) throws IOException { Text.writeString(out, wrappedInputFormatName); Text.writeString(out, wrappedSplits.get(0).getClass().getName()); out.writeInt(wrappedSplits.size()); - for(InputSplit split : wrappedSplits) { + for (InputSplit split : wrappedSplits) { writeWrappedSplit(split, out); } out.writeLong(length); - + if (locations == null || locations.length == 0) { out.writeInt(0); } else { @@ -120,37 +121,37 @@ public void readFields(DataInput in) throws IOException { Class clazz = null; try { clazz = (Class) - TezGroupedSplitsInputFormat.getClassFromName(inputSplitClassName); + TezGroupedSplitsInputFormat.getClassFromName(inputSplitClassName); } catch (TezException e) { throw new IOException(e); } int numSplits = in.readInt(); - + wrappedSplits = new ArrayList(numSplits); - for (int i=0; i 0) { locations = new String[numLocs]; - for (int i=0; i clazz) + + InputSplit readWrappedSplit(DataInput in, Class clazz) throws IOException { InputSplit split; try { @@ -161,7 +162,7 @@ InputSplit readWrappedSplit(DataInput in, Class clazz) split.readFields(in); return split; } - + @Override public long getLength() throws IOException { return length; @@ -181,7 +182,7 @@ public void setConf(Configuration conf) { public Configuration getConf() { return conf; } - + public String getRack() { return rack; } diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplitsInputFormat.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplitsInputFormat.java index 6266ec1bcf..c99d3cc704 100644 --- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplitsInputFormat.java +++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezGroupedSplitsInputFormat.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -51,9 +51,9 @@ */ @Public @Evolving -public class TezGroupedSplitsInputFormat - implements InputFormat, Configurable{ - +public class TezGroupedSplitsInputFormat + implements InputFormat, Configurable { + private static final Logger LOG = LoggerFactory.getLogger(TezGroupedSplitsInputFormat.class); InputFormat wrappedInputFormat; @@ -62,11 +62,11 @@ public class TezGroupedSplitsInputFormat SplitSizeEstimator estimator; SplitLocationProvider locationProvider; - + public TezGroupedSplitsInputFormat() { - + } - + public void setInputFormat(InputFormat wrappedInputFormat) { this.wrappedInputFormat = wrappedInputFormat; if (LOG.isDebugEnabled()) { @@ -83,13 +83,13 @@ public void setSplitLocationProvider(SplitLocationProvider locationProvider) { this.locationProvider = Objects.requireNonNull(locationProvider); LOG.debug("Split size location provider: {}", locationProvider); } - + public void setDesiredNumberOfSplits(int num) { Preconditions.checkArgument(num >= 0); this.desiredNumSplits = num; LOG.debug("desiredNumSplits: {}", desiredNumSplits); } - + @Override public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { InputSplit[] originalSplits = wrappedInputFormat.getSplits(job, numSplits); @@ -99,10 +99,10 @@ public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { .getGroupedSplits(conf, originalSplits, desiredNumSplits, wrappedInputFormatName, estimator, locationProvider); } - + @Override public RecordReader getRecordReader(InputSplit split, JobConf job, - Reporter reporter) throws IOException { + Reporter reporter) throws IOException { TezGroupedSplit groupedSplit = (TezGroupedSplit) split; try { initInputFormatFromSplit(groupedSplit); @@ -111,11 +111,11 @@ public RecordReader getRecordReader(InputSplit split, JobConf job, } return new TezGroupedSplitsRecordReader(groupedSplit, job, reporter); } - - @SuppressWarnings({ "unchecked", "rawtypes" }) + + @SuppressWarnings({"unchecked", "rawtypes"}) void initInputFormatFromSplit(TezGroupedSplit split) throws TezException { if (wrappedInputFormat == null) { - Class clazz = (Class) + Class clazz = (Class) getClassFromName(split.wrappedInputFormatName); try { wrappedInputFormat = org.apache.hadoop.util.ReflectionUtils.newInstance(clazz, conf); @@ -140,10 +140,10 @@ public class TezGroupedSplitsRecordReader implements RecordReader { final AtomicInteger initIndex; final int numReaders; final ExecutorService initReaderExecService; - final Queue>> initedReaders; + final Queue>> initedReaders; public TezGroupedSplitsRecordReader(TezGroupedSplit split, JobConf job, - Reporter reporter) throws IOException { + Reporter reporter) throws IOException { this.groupedSplit = split; this.job = job; this.reporter = reporter; @@ -189,7 +189,7 @@ private void preInitReaders() { })); } } - + @Override public boolean next(K key, V value) throws IOException { @@ -205,17 +205,17 @@ public boolean next(K key, V value) throws IOException { public K createKey() { return curReader.createKey(); } - + @Override public V createValue() { return curReader.createValue(); } - + @Override public float getProgress() throws IOException { - return Math.min(1.0f, getPos()/(float)(groupedSplit.getLength())); + return Math.min(1.0f, getPos() / (float) (groupedSplit.getLength())); } - + @Override public void close() throws IOException { if (curReader != null) { @@ -223,13 +223,13 @@ public void close() throws IOException { curReader = null; } } - + protected boolean initNextRecordReader() throws IOException { if (curReader != null) { curReader.close(); curReader = null; if (idx > 0) { - progress += groupedSplit.wrappedSplits.get(idx-1).getLength(); + progress += groupedSplit.wrappedSplits.get(idx - 1).getLength(); } } @@ -241,8 +241,8 @@ protected boolean initNextRecordReader() throws IOException { } if (LOG.isDebugEnabled()) { - LOG.debug("Init record reader for index " + idx + " of " + - groupedSplit.wrappedSplits.size()); + LOG.debug("Init record reader for index " + idx + " of " + + groupedSplit.wrappedSplits.size()); } // get a record reader for the idx-th chunk @@ -261,7 +261,7 @@ protected boolean initNextRecordReader() throws IOException { } private void cancelsFutures() { - for (Future> f : initedReaders) { + for (Future> f : initedReaders) { f.cancel(true); } } @@ -286,5 +286,4 @@ public void setConf(Configuration conf) { public Configuration getConf() { return conf; } - } diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezMapredSplitsGrouper.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezMapredSplitsGrouper.java index 38a213ec9a..b3a68e9765 100644 --- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezMapredSplitsGrouper.java +++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapred/split/TezMapredSplitsGrouper.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -49,8 +49,8 @@ public class TezMapredSplitsGrouper extends TezSplitGrouper { private static final Logger LOG = LoggerFactory.getLogger(TezMapredSplitsGrouper.class); public InputSplit[] getGroupedSplits(Configuration conf, - InputSplit[] originalSplits, int desiredNumSplits, - String wrappedInputFormatName) throws IOException { + InputSplit[] originalSplits, int desiredNumSplits, + String wrappedInputFormatName) throws IOException { return getGroupedSplits(conf, originalSplits, desiredNumSplits, wrappedInputFormatName, null); } @@ -62,10 +62,9 @@ public InputSplit[] getGroupedSplits(Configuration conf, estimator, null); } - public InputSplit[] getGroupedSplits(Configuration conf, - InputSplit[] originalSplits, int desiredNumSplits, - String wrappedInputFormatName, SplitSizeEstimator estimator, SplitLocationProvider locationProvider) throws IOException { + InputSplit[] originalSplits, int desiredNumSplits, + String wrappedInputFormatName, SplitSizeEstimator estimator, SplitLocationProvider locationProvider) throws IOException { Objects.requireNonNull(originalSplits, "Splits must be specified"); List originalSplitContainers = Lists.transform(Arrays.asList(originalSplits), @@ -94,7 +93,6 @@ public InputSplit apply(SplitContainer input) { } }); - return new TezGroupedSplit(underlyingSplits, input.getWrappedInputFormatName(), input.getLocations(), input.getRack(), input.getLength()); } diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReaderTez.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReaderTez.java index db156d2ce2..955a9446b2 100644 --- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReaderTez.java +++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReaderTez.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -49,7 +49,7 @@ public class SplitMetaInfoReaderTez { public static final byte[] META_SPLIT_FILE_HEADER = JobSplit.META_SPLIT_FILE_HEADER; private static FSDataInputStream getFSDataIS(Configuration conf, - FileSystem fs) throws IOException { + FileSystem fs) throws IOException { long maxMetaInfoSize = conf.getLong( MRJobConfig.SPLIT_METAINFO_MAXSIZE, MRJobConfig.DEFAULT_SPLIT_METAINFO_MAXSIZE); @@ -98,7 +98,7 @@ private static FSDataInputStream getFSDataIS(Configuration conf, // Forked from the MR variant so that the metaInfo file as well as the split // file can be read from local fs - relying on these files being localized. public static TaskSplitMetaInfo[] readSplitMetaInfo(Configuration conf, - FileSystem fs) throws IOException { + FileSystem fs) throws IOException { FSDataInputStream in = null; try { in = getFSDataIS(conf, fs); @@ -134,7 +134,7 @@ public static TaskSplitMetaInfo[] readSplitMetaInfo(Configuration conf, * @throws IOException */ public static TaskSplitMetaInfo getSplitMetaInfo(Configuration conf, - FileSystem fs, int index) throws IOException { + FileSystem fs, int index) throws IOException { FSDataInputStream in = null; try { in = getFSDataIS(conf, fs); diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitSizeEstimator.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitSizeEstimator.java index 2f8d74f3a7..634d270ac6 100644 --- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitSizeEstimator.java +++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/SplitSizeEstimator.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplit.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplit.java index 2d198ad8d4..c31a4d4524 100644 --- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplit.java +++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplit.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -42,8 +42,8 @@ */ @Public @Evolving -public class TezGroupedSplit extends InputSplit - implements Writable, Configurable { +public class TezGroupedSplit extends InputSplit + implements Writable, Configurable { List wrappedSplits = null; String wrappedInputFormatName = null; @@ -54,7 +54,7 @@ public class TezGroupedSplit extends InputSplit @InterfaceAudience.Private public TezGroupedSplit() { - + } @InterfaceAudience.Private @@ -71,18 +71,18 @@ public TezGroupedSplit(List wrappedSplits, String wrappedInputFormat } public TezGroupedSplit(int numSplits, String wrappedInputFormatName, - String[] locations, String rack) { + String[] locations, String rack) { this.wrappedSplits = new ArrayList(numSplits); this.wrappedInputFormatName = wrappedInputFormatName; this.locations = locations; this.rack = rack; } - + public TezGroupedSplit(int numSplits, String wrappedInputFormatName, - String[] locations) { + String[] locations) { this(numSplits, wrappedInputFormatName, locations, null); } - + public void addSplit(InputSplit split) { wrappedSplits.add(split); try { @@ -105,11 +105,11 @@ public void write(DataOutput out) throws IOException { Text.writeString(out, wrappedInputFormatName); Text.writeString(out, wrappedSplits.get(0).getClass().getName()); out.writeInt(wrappedSplits.size()); - for(InputSplit split : wrappedSplits) { + for (InputSplit split : wrappedSplits) { writeWrappedSplit(split, out); } out.writeLong(length); - + if (locations == null || locations.length == 0) { out.writeInt(0); } else { @@ -128,32 +128,32 @@ public void readFields(DataInput in) throws IOException { Class clazz = null; try { clazz = (Class) - TezGroupedSplitsInputFormat.getClassFromName(inputSplitClassName); + TezGroupedSplitsInputFormat.getClassFromName(inputSplitClassName); } catch (TezException e) { throw new IOException(e); } - + int numSplits = in.readInt(); - + wrappedSplits = new ArrayList(numSplits); - for (int i=0; i 0) { locations = new String[numLocs]; - for (int i=0; i clazz) { try { InputSplit split = clazz.newInstance(); @@ -171,13 +171,13 @@ InputSplit readWrappedSplit(DataInput in, Class clazz) { return split; } else { throw new TezUncheckedException( - split.getClass().getName() + " is not a Writable"); + split.getClass().getName() + " is not a Writable"); } } catch (Exception e) { throw new TezUncheckedException(e); } } - + @Override public long getLength() throws IOException, InterruptedException { return length; @@ -187,7 +187,7 @@ public long getLength() throws IOException, InterruptedException { public String[] getLocations() throws IOException, InterruptedException { return locations; } - + @Override public void setConf(Configuration conf) { this.conf = conf; @@ -197,7 +197,7 @@ public void setConf(Configuration conf) { public Configuration getConf() { return conf; } - + public String getRack() { return rack; } diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplitsInputFormat.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplitsInputFormat.java index b85e97e467..38ed6e3ff4 100644 --- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplitsInputFormat.java +++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezGroupedSplitsInputFormat.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -47,8 +47,8 @@ @Public @Evolving public class TezGroupedSplitsInputFormat extends InputFormat - implements Configurable{ - + implements Configurable { + private static final Logger LOG = LoggerFactory.getLogger(TezGroupedSplitsInputFormat.class); InputFormat wrappedInputFormat; @@ -56,18 +56,18 @@ public class TezGroupedSplitsInputFormat extends InputFormat Configuration conf; SplitSizeEstimator estimator; SplitLocationProvider locationProvider; - + public TezGroupedSplitsInputFormat() { - + } - + public void setInputFormat(InputFormat wrappedInputFormat) { this.wrappedInputFormat = wrappedInputFormat; if (LOG.isDebugEnabled()) { LOG.debug("wrappedInputFormat: " + wrappedInputFormat.getClass().getName()); } } - + public void setDesiredNumberOfSplits(int num) { Preconditions.checkArgument(num >= 0); this.desiredNumSplits = num; @@ -97,7 +97,7 @@ public List getSplits(JobContext context) throws IOException, @Override public RecordReader createRecordReader(InputSplit split, - TaskAttemptContext context) throws IOException, InterruptedException { + TaskAttemptContext context) throws IOException, InterruptedException { TezGroupedSplit groupedSplit = (TezGroupedSplit) split; try { initInputFormatFromSplit(groupedSplit); @@ -106,11 +106,11 @@ public RecordReader createRecordReader(InputSplit split, } return new TezGroupedSplitsRecordReader(groupedSplit, context); } - - @SuppressWarnings({ "rawtypes", "unchecked" }) + + @SuppressWarnings({"rawtypes", "unchecked"}) void initInputFormatFromSplit(TezGroupedSplit split) throws TezException { if (wrappedInputFormat == null) { - Class clazz = (Class) + Class clazz = (Class) getClassFromName(split.wrappedInputFormatName); try { wrappedInputFormat = org.apache.hadoop.util.ReflectionUtils.newInstance(clazz, conf); @@ -119,27 +119,27 @@ void initInputFormatFromSplit(TezGroupedSplit split) throws TezException { } } } - + static Class getClassFromName(String name) throws TezException { return ReflectionUtils.getClazz(name); } - - public class TezGroupedSplitsRecordReader extends RecordReader { + + public class TezGroupedSplitsRecordReader extends RecordReader { TezGroupedSplit groupedSplit; TaskAttemptContext context; int idx = 0; long progress; RecordReader curReader; - + public TezGroupedSplitsRecordReader(TezGroupedSplit split, - TaskAttemptContext context) throws IOException { + TaskAttemptContext context) throws IOException { this.groupedSplit = split; this.context = context; } - + public void initialize(InputSplit split, - TaskAttemptContext context) throws IOException, InterruptedException { + TaskAttemptContext context) throws IOException, InterruptedException { if (this.groupedSplit != split) { throw new TezUncheckedException("Splits dont match"); } @@ -148,7 +148,7 @@ public void initialize(InputSplit split, } initNextRecordReader(); } - + public boolean nextKeyValue() throws IOException, InterruptedException { while ((curReader == null) || !curReader.nextKeyValue()) { // false return finishes. true return loops back for nextKeyValue() @@ -162,25 +162,25 @@ public boolean nextKeyValue() throws IOException, InterruptedException { public K getCurrentKey() throws IOException, InterruptedException { return curReader.getCurrentKey(); } - + public V getCurrentValue() throws IOException, InterruptedException { return curReader.getCurrentValue(); } - + public void close() throws IOException { if (curReader != null) { curReader.close(); curReader = null; } } - + protected boolean initNextRecordReader() throws IOException { if (curReader != null) { curReader.close(); curReader = null; if (idx > 0) { try { - progress += groupedSplit.wrappedSplits.get(idx-1).getLength(); + progress += groupedSplit.wrappedSplits.get(idx - 1).getLength(); } catch (InterruptedException e) { throw new TezUncheckedException(e); } @@ -199,12 +199,12 @@ protected boolean initNextRecordReader() throws IOException { curReader.initialize(groupedSplit.wrappedSplits.get(idx), context); } catch (Exception e) { - throw new RuntimeException (e); + throw new RuntimeException(e); } idx++; return true; } - + /** * return progress based on the amount of data processed so far. */ @@ -215,7 +215,7 @@ public float getProgress() throws IOException, InterruptedException { subprogress = (long) (curReader.getProgress() * groupedSplit.wrappedSplits .get(idx - 1).getLength()); } - return Math.min(1.0f, (progress + subprogress)/(float)(groupedSplit.getLength())); + return Math.min(1.0f, (progress + subprogress) / (float) (groupedSplit.getLength())); } } @@ -228,5 +228,4 @@ public void setConf(Configuration conf) { public Configuration getConf() { return conf; } - } diff --git a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezMapReduceSplitsGrouper.java b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezMapReduceSplitsGrouper.java index 4f638e1400..7e7c7a0e3e 100644 --- a/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezMapReduceSplitsGrouper.java +++ b/tez-mapreduce/src/main/java/org/apache/hadoop/mapreduce/split/TezMapReduceSplitsGrouper.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -113,8 +113,8 @@ public class TezMapReduceSplitsGrouper extends TezSplitGrouper { * @deprecated See equivalent in {@link TezSplitGrouper} */ @Deprecated - public static final String TEZ_GROUPING_RACK_SPLIT_SIZE_REDUCTION = - TezSplitGrouper.TEZ_GROUPING_RACK_SPLIT_SIZE_REDUCTION; + public static final String TEZ_GROUPING_RACK_SPLIT_SIZE_REDUCTION = + TezSplitGrouper.TEZ_GROUPING_RACK_SPLIT_SIZE_REDUCTION; /** * @deprecated See equivalent in {@link TezSplitGrouper} */ @@ -132,10 +132,9 @@ public class TezMapReduceSplitsGrouper extends TezSplitGrouper { @Deprecated public static final boolean TEZ_GROUPING_REPEATABLE_DEFAULT = TezSplitGrouper.TEZ_GROUPING_REPEATABLE_DEFAULT; - public List getGroupedSplits(Configuration conf, - List originalSplits, int desiredNumSplits, - String wrappedInputFormatName) throws IOException, InterruptedException { + List originalSplits, int desiredNumSplits, + String wrappedInputFormatName) throws IOException, InterruptedException { return getGroupedSplits(conf, originalSplits, desiredNumSplits, wrappedInputFormatName, null); } @@ -163,7 +162,6 @@ public SplitContainer apply(InputSplit input) { } }); - return Lists.transform(super .getGroupedSplits(conf, originalSplitContainers, desiredNumSplits, wrappedInputFormatName, estimator == null ? null : @@ -184,7 +182,6 @@ public InputSplit apply(SplitContainer input) { return new TezGroupedSplit(underlyingSplits, input.getWrappedInputFormatName(), input.getLocations(), input.getRack(), input.getLength()); - } }); } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/dag/api/client/MRDAGClient.java b/tez-mapreduce/src/main/java/org/apache/tez/dag/api/client/MRDAGClient.java index c2646bd819..a0b19725a2 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/dag/api/client/MRDAGClient.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/dag/api/client/MRDAGClient.java @@ -103,7 +103,7 @@ public void close() throws IOException { @Override public DAGStatus getDAGStatus(@Nullable Set statusOptions, - long timeout) throws IOException, TezException { + long timeout) throws IOException, TezException { return getDAGStatus(statusOptions); } @@ -111,5 +111,4 @@ public DAGStatus getDAGStatus(@Nullable Set statusOptions, public String getWebUIAddress() throws IOException, TezException { throw new TezException("MRDAGClient.getWebUIAddress is not supported"); } - } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ClientCache.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ClientCache.java index f7c8e07629..e35f1a2786 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ClientCache.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ClientCache.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,7 +29,7 @@ public class ClientCache { private final Configuration conf; private final ResourceMgrDelegate rm; - private Map cache = + private Map cache = new HashMap(); public ClientCache(Configuration conf, ResourceMgrDelegate rm) { @@ -46,5 +46,4 @@ public synchronized ClientServiceDelegate getClient(JobID jobId) { } return client; } - } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ClientServiceDelegate.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ClientServiceDelegate.java index 3b9af9b44a..ed28f8c2b3 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ClientServiceDelegate.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ClientServiceDelegate.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -42,7 +42,7 @@ public class ClientServiceDelegate { // how to handle completed jobs that the RM does not know about? public ClientServiceDelegate(Configuration conf, ResourceMgrDelegate rm, - JobID jobId) { + JobID jobId) { this.conf = new TezConfiguration(conf); // Cloning for modifying. // For faster redirects from AM to HS. this.conf.setInt( @@ -60,7 +60,7 @@ public Counters getJobCounters(JobID jobId) } public TaskCompletionEvent[] getTaskCompletionEvents(JobID jobId, - int fromEventId, int maxEvents) + int fromEventId, int maxEvents) throws IOException, InterruptedException { // FIXME seems like there is support in client to query task failure // related information @@ -69,12 +69,12 @@ public TaskCompletionEvent[] getTaskCompletionEvents(JobID jobId, } public String[] getTaskDiagnostics(org.apache.hadoop.mapreduce.TaskAttemptID - taId) + taId) throws IOException, InterruptedException { // FIXME need support to query task diagnostics? return new String[0]; } - + public JobStatus getJobStatus(JobID oldJobID) throws IOException { // handled in YARNRunner throw new UnsupportedOperationException(); @@ -82,19 +82,19 @@ public JobStatus getJobStatus(JobID oldJobID) throws IOException { public TaskReport[] getTaskReports( JobID oldJobID, TaskType taskType) - throws IOException{ + throws IOException { // TEZ-146: need to return real task reports return new TaskReport[0]; } public boolean killTask(TaskAttemptID taskAttemptID, boolean fail) - throws IOException { + throws IOException { // FIXME need support to kill a task attempt? throw new UnsupportedOperationException(); } public boolean killJob(JobID oldJobID) - throws IOException { + throws IOException { // FIXME need support to kill a dag? // Should this be just an RM killApplication? // For one dag per AM, RM kill should suffice @@ -102,9 +102,9 @@ public boolean killJob(JobID oldJobID) } public LogParams getLogFilePath(JobID oldJobID, - TaskAttemptID oldTaskAttemptID) + TaskAttemptID oldTaskAttemptID) throws YarnException, IOException { // FIXME logs for an attempt? throw new UnsupportedOperationException(); - } + } } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/DAGJobStatus.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/DAGJobStatus.java index 9acd8368c6..1ef81fdcb3 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/DAGJobStatus.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/DAGJobStatus.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.mapreduce.client; @@ -137,7 +137,7 @@ public synchronized String getQueue() { @Override public synchronized float getMapProgress() { - if(dagStatus.getVertexProgress() != null) { + if (dagStatus.getVertexProgress() != null) { return getProgress(MultiStageMRConfigUtil.getInitialMapVertexName()); } if (dagStatus.getState() == DAGStatus.State.SUCCEEDED) { @@ -167,7 +167,7 @@ public synchronized float getSetupProgress() { @Override public synchronized float getReduceProgress() { - if(dagStatus.getVertexProgress() != null) { + if (dagStatus.getVertexProgress() != null) { return getProgress(MultiStageMRConfigUtil.getFinalReduceVertexName()); } if (dagStatus.getState() == DAGStatus.State.SUCCEEDED) { @@ -179,21 +179,21 @@ public synchronized float getReduceProgress() { @Override public synchronized State getState() { switch (dagStatus.getState()) { - case SUBMITTED: - case INITING: - return State.PREP; - case RUNNING: - return State.RUNNING; - case SUCCEEDED: - return State.SUCCEEDED; - case KILLED: - return State.KILLED; - case FAILED: - case ERROR: - return State.FAILED; - default: - throw new TezUncheckedException("Unknown value of DAGState.State:" - + dagStatus.getState()); + case SUBMITTED: + case INITING: + return State.PREP; + case RUNNING: + return State.RUNNING; + case SUCCEEDED: + return State.SUCCEEDED; + case KILLED: + return State.KILLED; + case FAILED: + case ERROR: + return State.FAILED; + default: + throw new TezUncheckedException("Unknown value of DAGState.State:" + + dagStatus.getState()); } } @@ -372,15 +372,14 @@ public String toString() { private float getProgress(String vertexName) { Progress progress = dagStatus.getVertexProgress().get(vertexName); - if(progress == null) { + if (progress == null) { // no such stage. return 0 like MR app currently does. return 0; } float totalTasks = (float) progress.getTotalTaskCount(); - if(totalTasks != 0) { - return progress.getSucceededTaskCount()/totalTasks; + if (totalTasks != 0) { + return progress.getSucceededTaskCount() / totalTasks; } return 0; } - } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/NotRunningJob.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/NotRunningJob.java index 8837df6818..b90fd63b69 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/NotRunningJob.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/NotRunningJob.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.mapreduce.client; @@ -70,12 +70,11 @@ public class NotRunningJob implements MRClientProtocol { private RecordFactory recordFactory = - RecordFactoryProvider.getRecordFactory(null); + RecordFactoryProvider.getRecordFactory(null); private final JobState jobState; private final ApplicationReport applicationReport; - private ApplicationReport getUnknownApplicationReport() { ApplicationId unknownAppId = recordFactory .newRecordInstance(ApplicationId.class); @@ -96,7 +95,7 @@ private ApplicationReport getUnknownApplicationReport() { NotRunningJob(ApplicationReport applicationReport, JobState jobState) { this.applicationReport = - (applicationReport == null) ? + (applicationReport == null) ? getUnknownApplicationReport() : applicationReport; this.jobState = jobState; } @@ -105,7 +104,7 @@ private ApplicationReport getUnknownApplicationReport() { public FailTaskAttemptResponse failTaskAttempt( FailTaskAttemptRequest request) throws IOException { FailTaskAttemptResponse resp = - recordFactory.newRecordInstance(FailTaskAttemptResponse.class); + recordFactory.newRecordInstance(FailTaskAttemptResponse.class); return resp; } @@ -113,7 +112,7 @@ public FailTaskAttemptResponse failTaskAttempt( public GetCountersResponse getCounters(GetCountersRequest request) throws IOException { GetCountersResponse resp = - recordFactory.newRecordInstance(GetCountersResponse.class); + recordFactory.newRecordInstance(GetCountersResponse.class); Counters counters = recordFactory.newRecordInstance(Counters.class); counters.addAllCounterGroups(new HashMap()); resp.setCounters(counters); @@ -124,7 +123,7 @@ public GetCountersResponse getCounters(GetCountersRequest request) public GetDiagnosticsResponse getDiagnostics(GetDiagnosticsRequest request) throws IOException { GetDiagnosticsResponse resp = - recordFactory.newRecordInstance(GetDiagnosticsResponse.class); + recordFactory.newRecordInstance(GetDiagnosticsResponse.class); resp.addDiagnostics(""); return resp; } @@ -133,7 +132,7 @@ public GetDiagnosticsResponse getDiagnostics(GetDiagnosticsRequest request) public GetJobReportResponse getJobReport(GetJobReportRequest request) throws IOException { JobReport jobReport = - recordFactory.newRecordInstance(JobReport.class); + recordFactory.newRecordInstance(JobReport.class); jobReport.setJobId(request.getJobId()); jobReport.setJobState(jobState); jobReport.setUser(applicationReport.getUser()); @@ -154,7 +153,7 @@ public GetTaskAttemptCompletionEventsResponse getTaskAttemptCompletionEvents( GetTaskAttemptCompletionEventsRequest request) throws IOException { GetTaskAttemptCompletionEventsResponse resp = - recordFactory.newRecordInstance(GetTaskAttemptCompletionEventsResponse.class); + recordFactory.newRecordInstance(GetTaskAttemptCompletionEventsResponse.class); resp.addAllCompletionEvents(new ArrayList()); return resp; } @@ -170,7 +169,7 @@ public GetTaskAttemptReportResponse getTaskAttemptReport( public GetTaskReportResponse getTaskReport(GetTaskReportRequest request) throws IOException { GetTaskReportResponse resp = - recordFactory.newRecordInstance(GetTaskReportResponse.class); + recordFactory.newRecordInstance(GetTaskReportResponse.class); TaskReport report = recordFactory.newRecordInstance(TaskReport.class); report.setTaskId(request.getTaskId()); report.setTaskState(TaskState.NEW); @@ -185,7 +184,7 @@ public GetTaskReportResponse getTaskReport(GetTaskReportRequest request) public GetTaskReportsResponse getTaskReports(GetTaskReportsRequest request) throws IOException { GetTaskReportsResponse resp = - recordFactory.newRecordInstance(GetTaskReportsResponse.class); + recordFactory.newRecordInstance(GetTaskReportsResponse.class); resp.addAllTaskReports(new ArrayList()); return resp; } @@ -194,7 +193,7 @@ public GetTaskReportsResponse getTaskReports(GetTaskReportsRequest request) public KillJobResponse killJob(KillJobRequest request) throws IOException { KillJobResponse resp = - recordFactory.newRecordInstance(KillJobResponse.class); + recordFactory.newRecordInstance(KillJobResponse.class); return resp; } @@ -202,7 +201,7 @@ public KillJobResponse killJob(KillJobRequest request) public KillTaskResponse killTask(KillTaskRequest request) throws IOException { KillTaskResponse resp = - recordFactory.newRecordInstance(KillTaskResponse.class); + recordFactory.newRecordInstance(KillTaskResponse.class); return resp; } @@ -210,7 +209,7 @@ public KillTaskResponse killTask(KillTaskRequest request) public KillTaskAttemptResponse killTaskAttempt( KillTaskAttemptRequest request) throws IOException { KillTaskAttemptResponse resp = - recordFactory.newRecordInstance(KillTaskAttemptResponse.class); + recordFactory.newRecordInstance(KillTaskAttemptResponse.class); return resp; } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ResourceMgrDelegate.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ResourceMgrDelegate.java index 09f7dd661b..5145c63260 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ResourceMgrDelegate.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/ResourceMgrDelegate.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.mapreduce.client; @@ -51,7 +51,7 @@ public class ResourceMgrDelegate { private static final Logger LOG = LoggerFactory.getLogger(ResourceMgrDelegate.class); - + private YarnConfiguration conf; private GetNewApplicationResponse application; private ApplicationId applicationId; @@ -108,7 +108,7 @@ public ClusterMetrics getClusterMetrics() throws IOException, } catch (YarnException e) { throw new IOException(e); } - ClusterMetrics oldMetrics = new ClusterMetrics(1, 1, 1, 1, 1, 1, + ClusterMetrics oldMetrics = new ClusterMetrics(1, 1, 1, 1, 1, 1, metrics.getNumNodeManagers() * 10, metrics.getNumNodeManagers() * 2, 1, metrics.getNumNodeManagers(), 0, 0); return oldMetrics; @@ -120,7 +120,7 @@ public Token getDelegationToken(Text renewer) throws IOException, try { // Remove rmAddress after YARN-868 is addressed return ConverterUtils.convertFromYarn( - client.getRMDelegationToken(renewer), rmAddress); + client.getRMDelegationToken(renewer), rmAddress); } catch (YarnException e) { throw new IOException(e); } @@ -132,7 +132,7 @@ public String getFilesystemName() throws IOException, InterruptedException { public JobID getNewJobID() throws IOException, InterruptedException { try { - this.application = + this.application = client.createApplication().getNewApplicationResponse(); } catch (YarnException e) { throw new IOException(e); @@ -142,13 +142,13 @@ public JobID getNewJobID() throws IOException, InterruptedException { } public QueueInfo getQueue(String queueName) throws IOException, - InterruptedException { + InterruptedException { try { org.apache.hadoop.yarn.api.records.QueueInfo queueInfo = client.getQueueInfo(queueName); return (queueInfo == null) ? null : TypeConverter.fromYarn(queueInfo, conf); - } catch (YarnException e) { + } catch (YarnException e) { throw new IOException(e); } } @@ -184,7 +184,7 @@ public QueueInfo[] getChildQueues(String parent) throws IOException, InterruptedException { try { return TypeConverter.fromYarnQueueInfo(client.getChildQueueInfos(parent), - this.conf); + this.conf); } catch (YarnException e) { throw new IOException(e); } @@ -192,32 +192,29 @@ public QueueInfo[] getChildQueues(String parent) throws IOException, public String getStagingAreaDir() throws IOException, InterruptedException { // Path path = new Path(MRJobConstants.JOB_SUBMIT_DIR); - String user = - UserGroupInformation.getCurrentUser().getShortUserName(); + String user = + UserGroupInformation.getCurrentUser().getShortUserName(); Path path = MRApps.getStagingAreaDir(conf, user); LOG.debug("getStagingAreaDir: dir=" + path); return path.toString(); } - public String getSystemDir() throws IOException, InterruptedException { Path sysDir = new Path(MRJobConfig.JOB_SUBMIT_DIR); //FileContext.getFileContext(conf).delete(sysDir, true); return sysDir.toString(); } - public long getTaskTrackerExpiryInterval() throws IOException, InterruptedException { return 0; } - + public void setJobPriority(JobID arg0, String arg1) throws IOException, InterruptedException { return; } - public long getProtocolVersion(String arg0, long arg1) throws IOException { return 0; } @@ -230,5 +227,4 @@ public void killApplication(ApplicationId appId) throws YarnException, IOException { client.killApplication(appId); } - } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java index 9dba357951..177489d726 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.mapreduce.client; @@ -123,7 +123,7 @@ /** * This class enables the current JobClient (0.22 hadoop) to run on YARN-TEZ. */ -@SuppressWarnings({ "unchecked" }) +@SuppressWarnings({"unchecked"}) @LimitedPrivate("Mapreduce") public class YARNRunner implements ClientProtocol { @@ -158,7 +158,7 @@ public YARNRunner(Configuration conf) { * @param resMgrDelegate the resourcemanager client handle. */ public YARNRunner(Configuration conf, ResourceMgrDelegate resMgrDelegate) { - this(conf, resMgrDelegate, new ClientCache(conf, resMgrDelegate)); + this(conf, resMgrDelegate, new ClientCache(conf, resMgrDelegate)); } /** @@ -169,14 +169,13 @@ public YARNRunner(Configuration conf, ResourceMgrDelegate resMgrDelegate) { * @param clientCache the client cache object. */ public YARNRunner(Configuration conf, ResourceMgrDelegate resMgrDelegate, - ClientCache clientCache) { + ClientCache clientCache) { this.conf = conf; this.tezConf = new TezConfiguration(conf); try { this.resMgrDelegate = resMgrDelegate; this.clientCache = clientCache; this.defaultFileContext = FileContext.getFileContext(this.conf); - } catch (UnsupportedFileSystemException ufe) { throw new RuntimeException("Error in instantiating YarnClient", ufe); } @@ -321,7 +320,7 @@ private Map createJobLocalResources( } // TODO gross hack - for (String s : new String[] { + for (String s : new String[]{ MRJobConfig.JOB_SPLIT, MRJobConfig.JOB_SPLIT_METAINFO}) { localResources.put(s, @@ -337,8 +336,8 @@ private Map createJobLocalResources( // FIXME isn't this a nice mess of a client? // read input, write splits, read splits again private List getMapLocationHintsFromInputSplits(JobID jobId, - FileSystem fs, Configuration conf, - String jobSubmitDir) throws IOException { + FileSystem fs, Configuration conf, + String jobSubmitDir) throws IOException { TaskSplitMetaInfo[] splitsInfo = SplitMetaInfoReader.readSplitMetaInfo(jobId, fs, conf, new Path(jobSubmitDir)); @@ -357,27 +356,27 @@ private List getMapLocationHintsFromInputSplits(JobID jobId, } private void setupMapReduceEnv(Configuration jobConf, - Map environment, boolean isMap) throws IOException { + Map environment, boolean isMap) throws IOException { if (isMap) { warnForJavaLibPath( - jobConf.get(MRJobConfig.MAP_JAVA_OPTS,""), + jobConf.get(MRJobConfig.MAP_JAVA_OPTS, ""), "map", MRJobConfig.MAP_JAVA_OPTS, MRJobConfig.MAP_ENV); warnForJavaLibPath( - jobConf.get(MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS,""), + jobConf.get(MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS, ""), "map", MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS, MRJobConfig.MAPRED_ADMIN_USER_ENV); } else { warnForJavaLibPath( - jobConf.get(MRJobConfig.REDUCE_JAVA_OPTS,""), + jobConf.get(MRJobConfig.REDUCE_JAVA_OPTS, ""), "reduce", MRJobConfig.REDUCE_JAVA_OPTS, MRJobConfig.REDUCE_ENV); warnForJavaLibPath( - jobConf.get(MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS,""), + jobConf.get(MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS, ""), "reduce", MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS, MRJobConfig.MAPRED_ADMIN_USER_ENV); @@ -387,8 +386,8 @@ private void setupMapReduceEnv(Configuration jobConf, } private Vertex createVertexForStage(Configuration stageConf, - Map jobLocalResources, - List locations, int stageNum, int totalStages) + Map jobLocalResources, + List locations, int stageNum, int totalStages) throws IOException { // stageNum starts from 0, goes till numStages - 1 boolean isMap = false; @@ -414,9 +413,9 @@ private Vertex createVertexForStage(Configuration stageConf, Resource taskResource = isMap ? MRHelpers.getResourceForMRMapper(stageConf) : MRHelpers.getResourceForMRReducer(stageConf); - + stageConf.set(MRJobConfig.MROUTPUT_FILE_NAME_PREFIX, "part"); - + UserPayload vertexUserPayload = TezUtils.createUserPayloadFromConf(stageConf); Vertex vertex = Vertex.create(vertexName, ProcessorDescriptor.create(processorName).setUserPayload(vertexUserPayload), @@ -431,7 +430,7 @@ private Vertex createVertexForStage(Configuration stageConf, configureMRInputWithLegacySplitsGenerated(stageConf, true)); } // Map only jobs. - if (stageNum == totalStages -1) { + if (stageNum == totalStages - 1) { OutputDescriptor od = OutputDescriptor.create(MROutputLegacy.class.getName()) .setUserPayload(vertexUserPayload); if (stageConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT, @@ -458,18 +457,18 @@ private Vertex createVertexForStage(Configuration stageConf, .addTaskLocalFiles(taskLocalResources) .setLocationHint(VertexLocationHint.create(locations)) .setTaskLaunchCmdOpts(taskJavaOpts); - + if (!isMap) { vertex.setVertexManagerPlugin((ShuffleVertexManager.createConfigBuilder(stageConf).build())); } if (LOG.isDebugEnabled()) { LOG.debug("Adding vertex to DAG" + ", vertexName=" - + vertex.getName() + ", processor=" - + vertex.getProcessorDescriptor().getClassName() + ", parallelism=" - + vertex.getParallelism() + ", javaOpts=" + vertex.getTaskLaunchCmdOpts() - + ", resources=" + vertex.getTaskResource() - // TODO Add localResources and Environment + + vertex.getName() + ", processor=" + + vertex.getProcessorDescriptor().getClassName() + ", parallelism=" + + vertex.getParallelism() + ", javaOpts=" + vertex.getTaskLaunchCmdOpts() + + ", resources=" + vertex.getTaskResource() + // TODO Add localResources and Environment ); } @@ -477,8 +476,8 @@ private Vertex createVertexForStage(Configuration stageConf, } private DAG createDAG(FileSystem fs, JobID jobId, Configuration[] stageConfs, - String jobSubmitDir, Credentials ts, - Map jobLocalResources) throws IOException { + String jobSubmitDir, Credentials ts, + Map jobLocalResources) throws IOException { String jobName = stageConfs[0].get(MRJobConfig.JOB_NAME, YarnConfiguration.DEFAULT_APPLICATION_NAME); @@ -504,7 +503,7 @@ private DAG createDAG(FileSystem fs, JobID jobId, Configuration[] stageConfs, // Set edge conf based on Input conf (compression etc properties for MapReduce are // w.r.t Outputs - MAP_OUTPUT_COMPRESS for example) Map partitionerConf = null; - if (stageConfs[i-1] != null) { + if (stageConfs[i - 1] != null) { partitionerConf = Maps.newHashMap(); for (Map.Entry entry : stageConfs[i - 1]) { partitionerConf.put(entry.getKey(), entry.getValue()); @@ -512,16 +511,15 @@ private DAG createDAG(FileSystem fs, JobID jobId, Configuration[] stageConfs, } OrderedPartitionedKVEdgeConfig edgeConf = OrderedPartitionedKVEdgeConfig.newBuilder(stageConfs[i - 1].get( - TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS), - stageConfs[i - 1].get(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS), - MRPartitioner.class.getName(), partitionerConf) - .setFromConfigurationUnfiltered(stageConfs[i-1]) + TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS), + stageConfs[i - 1].get(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS), + MRPartitioner.class.getName(), partitionerConf) + .setFromConfigurationUnfiltered(stageConfs[i - 1]) .configureInput().useLegacyInput().done() .build(); Edge edge = Edge.create(vertices[i - 1], vertices[i], edgeConf.createDefaultEdgeProperty()); dag.addEdge(edge); } - } return dag; } @@ -547,7 +545,7 @@ private TezConfiguration getDAGAMConfFromMRConf() { @Override public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts) - throws IOException, InterruptedException { + throws IOException, InterruptedException { ApplicationId appId = resMgrDelegate.getApplicationId(); @@ -623,20 +621,20 @@ public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts) try { dagAMConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, jobSubmitDir); - + // Set Tez parameters based on MR parameters. String queueName = jobConf.get(JobContext.QUEUE_NAME, YarnConfiguration.DEFAULT_QUEUE_NAME); dagAMConf.set(TezConfiguration.TEZ_QUEUE_NAME, queueName); - + int amMemMB = jobConf.getInt(MRJobConfig.MR_AM_VMEM_MB, MRJobConfig.DEFAULT_MR_AM_VMEM_MB); int amCores = jobConf.getInt(MRJobConfig.MR_AM_CPU_VCORES, MRJobConfig.DEFAULT_MR_AM_CPU_VCORES); dagAMConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, amMemMB); dagAMConf.setInt(TezConfiguration.TEZ_AM_RESOURCE_CPU_VCORES, amCores); - dagAMConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, + dagAMConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, jobConf.getInt(MRJobConfig.MR_AM_MAX_ATTEMPTS, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS)); - + tezClient = new MRTezClient("MapReduce", dagAMConf, false, jobLocalResources, ts); tezClient.start(); tezClient.submitDAGApplication(appId, dag); @@ -649,7 +647,7 @@ public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts) } private LocalResource createApplicationResource(FileContext fs, Path p, - LocalResourceType type) throws IOException { + LocalResourceType type) throws IOException { LocalResource rsrc = Records.newRecord(LocalResource.class); FileStatus rsrcStat = fs.getFileStatus(p); rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs @@ -683,7 +681,6 @@ public long renewDelegationToken(Token arg0) throw new UnsupportedOperationException("Use Token.renew instead"); } - @Override public Counters getJobCounters(JobID arg0) throws IOException, InterruptedException { @@ -702,7 +699,7 @@ public JobStatus getJobStatus(JobID jobID) throws IOException, String jobFile = MRApps.getJobFile(conf, user, jobID); DAGStatus dagStatus; try { - if(dagClient == null) { + if (dagClient == null) { dagClient = MRTezClient.getDAGClient(TypeConverter.toYarn(jobID).getAppId(), tezConf, null); } dagStatus = dagClient.getDAGStatus(null); @@ -714,7 +711,7 @@ public JobStatus getJobStatus(JobID jobID) throws IOException, @Override public TaskCompletionEvent[] getTaskCompletionEvents(JobID arg0, int arg1, - int arg2) throws IOException, InterruptedException { + int arg2) throws IOException, InterruptedException { return clientCache.getClient(arg0).getTaskCompletionEvents(arg0, arg1, arg2); } @@ -726,7 +723,7 @@ public String[] getTaskDiagnostics(TaskAttemptID arg0) throws IOException, @Override public TaskReport[] getTaskReports(JobID jobID, TaskType taskType) - throws IOException, InterruptedException { + throws IOException, InterruptedException { return clientCache.getClient(jobID) .getTaskReports(jobID, taskType); } @@ -765,7 +762,7 @@ public JobTrackerStatus getJobTrackerStatus() throws IOException, @Override public ProtocolSignature getProtocolSignature(String protocol, - long clientVersion, int clientMethodsHash) throws IOException { + long clientVersion, int clientMethodsHash) throws IOException { return ProtocolSignature.getProtocolSignature(this, protocol, clientVersion, clientMethodsHash); } @@ -781,13 +778,13 @@ public LogParams getLogFileParams(JobID jobID, TaskAttemptID taskAttemptID) } private static void warnForJavaLibPath(String opts, String component, - String javaConf, String envConf) { + String javaConf, String envConf) { if (opts != null && opts.contains("-Djava.library.path")) { LOG.warn("Usage of -Djava.library.path in " + javaConf + " can cause " + - "programs to no longer function if hadoop native libraries " + - "are used. These values should be set as part of the " + - "LD_LIBRARY_PATH in the " + component + " JVM env using " + - envConf + " config settings."); + "programs to no longer function if hadoop native libraries " + + "are used. These values should be set as part of the " + + "LD_LIBRARY_PATH in the " + component + " JVM env using " + + envConf + " config settings."); } } @@ -798,7 +795,7 @@ private static DataSourceDescriptor configureMRInputWithLegacySplitsGenerated(Co try { inputDescriptor = InputDescriptor.create(useLegacyInput ? MRInputLegacy.class - .getName() : MRInput.class.getName()) + .getName() : MRInput.class.getName()) .setUserPayload(MRInputHelpersInternal.createMRInputPayload(conf, null)); } catch (IOException e) { throw new TezUncheckedException(e); @@ -816,11 +813,10 @@ private static DataSourceDescriptor configureMRInputWithLegacySplitsGenerated(Co private static class MRInputHelpersInternal extends MRInputHelpers { protected static UserPayload createMRInputPayload(Configuration conf, - MRRuntimeProtos.MRSplitsProto mrSplitsProto) throws - IOException { + MRRuntimeProtos.MRSplitsProto mrSplitsProto) throws + IOException { return MRInputHelpers.createMRInputPayload(conf, mrSplitsProto, false, true); } } - } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YarnTezClientProtocolProvider.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YarnTezClientProtocolProvider.java index b77dec335d..55d994f7b5 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YarnTezClientProtocolProvider.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YarnTezClientProtocolProvider.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -48,5 +48,4 @@ public ClientProtocol create(InetSocketAddress addr, Configuration conf) public void close(ClientProtocol clientProtocol) throws IOException { // nothing to do } - } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/package-info.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/package-info.java index 4366a673e5..d8bab2bcb7 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/package-info.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/package-info.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,4 +19,4 @@ @Private package org.apache.tez.mapreduce.client; -import org.apache.hadoop.classification.InterfaceAudience.Private; \ No newline at end of file +import org.apache.hadoop.classification.InterfaceAudience.Private; diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java index adfd24dda6..a15a0b1d7d 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/combine/MRCombiner.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -65,16 +65,16 @@ public class MRCombiner implements Combiner { private static Logger LOG = LoggerFactory.getLogger(MRCombiner.class); - + private final Configuration conf; private final Class keyClass; private final Class valClass; private final RawComparator comparator; private final boolean useNewApi; - + private final TezCounter combineInputRecordsCounter; private final TezCounter combineOutputRecordsCounter; - + private final MRTaskReporter reporter; private final TaskAttemptID mrTaskAttemptID; @@ -87,30 +87,30 @@ public MRCombiner(TaskContext taskContext) throws IOException { conf = userConf; } - assert(taskContext instanceof InputContext || taskContext instanceof OutputContext); + assert (taskContext instanceof InputContext || taskContext instanceof OutputContext); if (taskContext instanceof OutputContext) { this.keyClass = ConfigUtils.getIntermediateOutputKeyClass(conf); this.valClass = ConfigUtils.getIntermediateOutputValueClass(conf); this.comparator = ConfigUtils.getIntermediateOutputKeyComparator(conf); - this.reporter = new MRTaskReporter((OutputContext)taskContext); + this.reporter = new MRTaskReporter((OutputContext) taskContext); } else { this.keyClass = ConfigUtils.getIntermediateInputKeyClass(conf); this.valClass = ConfigUtils.getIntermediateInputValueClass(conf); this.comparator = ConfigUtils.getIntermediateInputKeyComparator(conf); - this.reporter = new MRTaskReporter((InputContext)taskContext); + this.reporter = new MRTaskReporter((InputContext) taskContext); } combineInputRecordsCounter = taskContext.getCounters().findCounter(TaskCounter.COMBINE_INPUT_RECORDS); combineOutputRecordsCounter = taskContext.getCounters().findCounter(TaskCounter.COMBINE_OUTPUT_RECORDS); - - boolean isMap = conf.getBoolean(MRConfig.IS_MAP_PROCESSOR,false); + + boolean isMap = conf.getBoolean(MRConfig.IS_MAP_PROCESSOR, false); this.mrTaskAttemptID = new TaskAttemptID( new TaskID(String.valueOf(taskContext.getApplicationId() .getClusterTimestamp()), taskContext.getApplicationId().getId(), isMap ? TaskType.MAP : TaskType.REDUCE, taskContext.getTaskIndex()), taskContext.getTaskAttemptNumber()); - - LOG.info("Using combineKeyClass: " + keyClass + ", combineValueClass: " + valClass + ", combineComparator: " +comparator + ", useNewApi: " + useNewApi); + + LOG.info("Using combineKeyClass: " + keyClass + ", combineValueClass: " + valClass + ", combineComparator: " + comparator + ", useNewApi: " + useNewApi); } @Override @@ -121,16 +121,15 @@ public void combine(TezRawKeyValueIterator rawIter, Writer writer) } else { runOldCombiner(rawIter, writer); } - } ///////////////// Methods for old API ////////////////////// - + private void runOldCombiner(final TezRawKeyValueIterator rawIter, final Writer writer) throws IOException { Class reducerClazz = (Class) conf.getClass("mapred.combiner.class", null, Reducer.class); - + Reducer combiner = ReflectionUtils.newInstance(reducerClazz, conf); - + OutputCollector collector = new OutputCollector() { @Override public void collect(Object key, Object value) throws IOException { @@ -138,29 +137,29 @@ public void collect(Object key, Object value) throws IOException { combineOutputRecordsCounter.increment(1); } }; - + CombinerValuesIterator values = new CombinerValuesIterator(rawIter, keyClass, valClass, comparator); - + while (values.moveToNext()) { combiner.reduce(values.getKey(), values.getValues().iterator(), collector, reporter); } } - - private final class CombinerValuesIterator extends ValuesIterator { + + private final class CombinerValuesIterator extends ValuesIterator { public CombinerValuesIterator(TezRawKeyValueIterator rawIter, - Class keyClass, Class valClass, - RawComparator comparator) throws IOException { + Class keyClass, Class valClass, + RawComparator comparator) throws IOException { super(rawIter, comparator, keyClass, valClass, conf, null, combineInputRecordsCounter); } } - + ///////////////// End of methods for old API ////////////////////// - + ///////////////// Methods for new API ////////////////////// - + private void runNewCombiner(final TezRawKeyValueIterator rawIter, final Writer writer) throws InterruptedException, IOException { - + RecordWriter recordWriter = new RecordWriter() { @Override @@ -176,12 +175,12 @@ public void close(TaskAttemptContext context) throws IOException, // Will be closed by whoever invokes the combiner. } }; - + Class reducerClazz = (Class) conf .getClass(MRJobConfig.COMBINE_CLASS_ATTR, null, org.apache.hadoop.mapreduce.Reducer.class); org.apache.hadoop.mapreduce.Reducer reducer = ReflectionUtils.newInstance(reducerClazz, conf); - + org.apache.hadoop.mapreduce.Reducer.Context reducerContext = createReduceContext( conf, @@ -191,10 +190,10 @@ public void close(TaskAttemptContext context) throws IOException, new MRCounters.MRCounter(combineOutputRecordsCounter), recordWriter, reporter, - (RawComparator)comparator, + (RawComparator) comparator, keyClass, valClass); - + reducer.run(reducerContext); recordWriter.close(reducerContext); } @@ -248,7 +247,4 @@ public void close() throws IOException { .getReducerContext(rContext); return reducerContext; } - - - } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java index 71e5681cbf..c5326e3bea 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/committer/MROutputCommitter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -73,7 +73,7 @@ public void initialize() throws IOException { jobConf = new JobConf( TezUtils.createConfFromUserPayload(userPayload)); } - + // Read all credentials into the credentials instance stored in JobConf. jobConf.getCredentials().mergeAll(UserGroupInformation.getCurrentUser().getCredentials()); jobConf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, @@ -111,12 +111,12 @@ public void abortOutput(VertexStatus.State finalState) throws IOException { @SuppressWarnings("rawtypes") private org.apache.hadoop.mapreduce.OutputCommitter - getOutputCommitter(OutputCommitterContext context) { + getOutputCommitter(OutputCommitterContext context) { org.apache.hadoop.mapreduce.OutputCommitter committer = null; newApiCommitter = false; if (jobConf.getBoolean("mapred.reducer.new-api", false) - || jobConf.getBoolean("mapred.mapper.new-api", false)) { + || jobConf.getBoolean("mapred.mapper.new-api", false)) { newApiCommitter = true; } LOG.info("Committer for " + getContext().getVertexName() + ":" + getContext().getOutputName() + @@ -161,7 +161,7 @@ private JobContext getJobContextFromVertexContext(OutputCommitterContext context } private JobStatus.State getJobStateFromVertexStatusState(VertexStatus.State state) { - switch(state) { + switch (state) { case INITED: return JobStatus.State.PREP; case RUNNING: @@ -184,7 +184,6 @@ private static class MRJobContextImpl public MRJobContextImpl(JobConf jobConf, JobID jobId) { super(jobConf, jobId); } - } @SuppressWarnings("deprecation") @@ -203,7 +202,7 @@ public void recoverTask(int taskIndex, int attemptId) throws IOException { } TaskAttemptID taskAttemptID = new TaskAttemptID( Long.toString(getContext().getApplicationId().getClusterTimestamp()) - + String.valueOf(getContext().getVertexIndex()), + + String.valueOf(getContext().getVertexIndex()), getContext().getApplicationId().getId(), ((jobConf.getBoolean(MRConfig.IS_MAP_PROCESSOR, false) ? TaskType.MAP : TaskType.REDUCE)), @@ -212,5 +211,4 @@ public void recoverTask(int taskIndex, int attemptId) throws IOException { taskAttemptID); committer.recoverTask(taskContext); } - } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputAMSplitGenerator.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputAMSplitGenerator.java index d06a5f46a0..e92824535a 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputAMSplitGenerator.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputAMSplitGenerator.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -60,7 +60,7 @@ public class MRInputAMSplitGenerator extends InputInitializer { private boolean sendSerializedEvents; - + private static final Logger LOG = LoggerFactory.getLogger(MRInputAMSplitGenerator.class); public MRInputAMSplitGenerator( @@ -81,7 +81,7 @@ public List initialize() throws Exception { sw.reset().start(); Configuration conf = new JobConf(getContext().getVertexConfiguration()); TezUtils.addToConfFromByteString(conf, userPayloadProto.getConfigurationBytes()); - + sendSerializedEvents = conf.getBoolean( MRJobConfig.MR_TEZ_INPUT_INITIALIZER_SERIALIZE_EVENT_PAYLOAD, MRJobConfig.MR_TEZ_INPUT_INITIALIZER_SERIALIZE_EVENT_PAYLOAD_DEFAULT); @@ -101,9 +101,7 @@ public List initialize() throws Exception { TezSplitGrouper.TEZ_GROUPING_SPLIT_WAVES, TezSplitGrouper.TEZ_GROUPING_SPLIT_WAVES_DEFAULT); - int numTasks = (int)((totalResource*waves)/taskResource); - - + int numTasks = (int) ((totalResource * waves) / taskResource); boolean groupSplits = userPayloadProto.getGroupingEnabled(); boolean sortSplits = userPayloadProto.getSortSplitsEnabled(); @@ -127,7 +125,7 @@ public List initialize() throws Exception { List events = Lists.newArrayListWithCapacity(inputSplitInfo .getNumTasks() + 1); - + InputConfigureVertexTasksEvent configureVertexEvent = InputConfigureVertexTasksEvent.create( inputSplitInfo.getNumTasks(), VertexLocationHint.create(inputSplitInfo.getTaskLocationHints()), @@ -160,7 +158,7 @@ public List initialize() throws Exception { } } } - + return events; } @@ -168,5 +166,4 @@ public List initialize() throws Exception { public void handleInputInitializerEvent(List events) throws Exception { throw new UnsupportedOperationException("Not expecting to handle any events"); } - } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputSplitDistributor.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputSplitDistributor.java index c60e5d4c1a..7d371dc2bf 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputSplitDistributor.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputSplitDistributor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -59,7 +59,7 @@ public class MRInputSplitDistributor extends InputInitializer { private static final Logger LOG = LoggerFactory.getLogger(MRInputSplitDistributor.class); - + private boolean sendSerializedEvents; private MRSplitsProto splitsProto; @@ -88,7 +88,7 @@ public List initialize() throws IOException { LOG.info("Emitting serialized splits: " + sendSerializedEvents); this.splitsProto = userPayloadProto.getSplits(); - + MRInputUserPayloadProto.Builder updatedPayloadBuilder = MRInputUserPayloadProto.newBuilder(userPayloadProto); updatedPayloadBuilder.clearSplits(); diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/Utils.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/Utils.java index 050ba7912b..e968c58b0f 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/Utils.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/Utils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -61,5 +61,4 @@ public static Counter getMRCounter(TezCounter tezCounter) { Objects.requireNonNull(tezCounter); return new MRCounters.MRCounter(tezCounter); } - } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/GroupedSplitContainer.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/GroupedSplitContainer.java index c2362576d0..8ffaae60d2 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/GroupedSplitContainer.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/GroupedSplitContainer.java @@ -14,14 +14,12 @@ package org.apache.tez.mapreduce.grouper; - import java.util.List; import com.google.common.collect.Lists; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.tez.dag.api.TezUncheckedException; - /** * An entity to hold grouped splits - either mapred or mapreduce. */ @@ -42,7 +40,6 @@ public GroupedSplitContainer(int numSplits, String wrappedInputFormatName, this.rack = rack; } - public void addSplit(SplitContainer splitContainer) { wrappedSplits.add(splitContainer); try { diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/MapReduceSplitContainer.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/MapReduceSplitContainer.java index fc6a424bd3..a1432fa63d 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/MapReduceSplitContainer.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/MapReduceSplitContainer.java @@ -54,7 +54,6 @@ public boolean equals(Object o) { MapReduceSplitContainer that = (MapReduceSplitContainer) o; return !(inputSplit != null ? !inputSplit.equals(that.inputSplit) : that.inputSplit != null); - } @Override diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/MapredSplitContainer.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/MapredSplitContainer.java index ce86ad52ef..ae540e06f6 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/MapredSplitContainer.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/MapredSplitContainer.java @@ -54,7 +54,6 @@ public boolean equals(Object o) { MapredSplitContainer that = (MapredSplitContainer) o; return !(inputSplit != null ? !inputSplit.equals(that.inputSplit) : that.inputSplit != null); - } @Override diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/SplitContainer.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/SplitContainer.java index 383b9ca9c8..61c3c459bc 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/SplitContainer.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/SplitContainer.java @@ -26,7 +26,6 @@ public abstract class SplitContainer { private boolean isProcessed = false; - public abstract String[] getPreferredLocations() throws IOException, InterruptedException; public abstract long getLength() throws IOException, InterruptedException; diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/SplitLocationProviderWrapperMapred.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/SplitLocationProviderWrapperMapred.java index 89a15bab6e..9758c1ef9d 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/SplitLocationProviderWrapperMapred.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/SplitLocationProviderWrapperMapred.java @@ -31,7 +31,7 @@ public SplitLocationProviderWrapperMapred(SplitLocationProvider locationProvider @Override public String[] getPreferredLocations(SplitContainer rawContainer) throws IOException, InterruptedException { - MapredSplitContainer splitContainer = (MapredSplitContainer)rawContainer; + MapredSplitContainer splitContainer = (MapredSplitContainer) rawContainer; return locationProvider.getLocations(splitContainer.getRawSplit()); } } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/SplitSizeEstimatorWrapper.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/SplitSizeEstimatorWrapper.java index ebb33adb4b..0beacc92d1 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/SplitSizeEstimatorWrapper.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/SplitSizeEstimatorWrapper.java @@ -18,7 +18,6 @@ import org.apache.hadoop.classification.InterfaceAudience; - /** * An interface to handle split size estimation across mapred and mapreduce splits */ @@ -26,5 +25,4 @@ public interface SplitSizeEstimatorWrapper { long getEstimatedSize(SplitContainer splitContainer) throws IOException, InterruptedException; - } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/SplitSizeEstimatorWrapperMapReduce.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/SplitSizeEstimatorWrapperMapReduce.java index df6e3c9759..d2b161c0e2 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/SplitSizeEstimatorWrapperMapReduce.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/SplitSizeEstimatorWrapperMapReduce.java @@ -29,7 +29,7 @@ public SplitSizeEstimatorWrapperMapReduce(SplitSizeEstimator estimator) { @Override public long getEstimatedSize(SplitContainer rawContainer) throws IOException, InterruptedException { - MapReduceSplitContainer splitContainer = (MapReduceSplitContainer)rawContainer; + MapReduceSplitContainer splitContainer = (MapReduceSplitContainer) rawContainer; return estimator.getEstimatedSize(splitContainer.getRawSplit()); } -} \ No newline at end of file +} diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/SplitSizeEstimatorWrapperMapred.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/SplitSizeEstimatorWrapperMapred.java index 6dd3a56e61..6880fc4168 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/SplitSizeEstimatorWrapperMapred.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/SplitSizeEstimatorWrapperMapred.java @@ -29,7 +29,7 @@ public SplitSizeEstimatorWrapperMapred(SplitSizeEstimator estimator) { @Override public long getEstimatedSize(SplitContainer rawContainer) throws IOException, InterruptedException { - MapredSplitContainer splitContainer = (MapredSplitContainer)rawContainer; + MapredSplitContainer splitContainer = (MapredSplitContainer) rawContainer; return estimator.getEstimatedSize(splitContainer.getRawSplit()); } } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/TezSplitGrouper.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/TezSplitGrouper.java index 3b2f17d1ff..f17da1b8df 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/TezSplitGrouper.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/grouper/TezSplitGrouper.java @@ -69,13 +69,13 @@ public abstract class TezSplitGrouper { * Upper bound on the size (in bytes) of a grouped split, to avoid generating excessively large splits. */ public static final String TEZ_GROUPING_SPLIT_MAX_SIZE = "tez.grouping.max-size"; - public static final long TEZ_GROUPING_SPLIT_MAX_SIZE_DEFAULT = 1024*1024*1024L; + public static final long TEZ_GROUPING_SPLIT_MAX_SIZE_DEFAULT = 1024 * 1024 * 1024L; /** * Lower bound on the size (in bytes) of a grouped split, to avoid generating too many small splits. */ public static final String TEZ_GROUPING_SPLIT_MIN_SIZE = "tez.grouping.min-size"; - public static final long TEZ_GROUPING_SPLIT_MIN_SIZE_DEFAULT = 50*1024*1024L; + public static final long TEZ_GROUPING_SPLIT_MIN_SIZE_DEFAULT = 50 * 1024 * 1024L; /** * This factor is used to decrease the per group desired (length and count) limits for groups @@ -117,12 +117,15 @@ public abstract class TezSplitGrouper { static class LocationHolder { List splits; int headIndex = 0; + LocationHolder(int capacity) { splits = new ArrayList(capacity); } + boolean isEmpty() { return (headIndex == splits.size()); } + SplitContainer getUnprocessedHeadSplit() { while (!isEmpty()) { SplitContainer holder = splits.get(headIndex); @@ -133,6 +136,7 @@ SplitContainer getUnprocessedHeadSplit() { } return null; } + void incrementHeadIndex() { headIndex++; } @@ -168,8 +172,6 @@ Map createLocationsMap(Configuration conf) { return new HashMap(); } - - public List getGroupedSplits(Configuration conf, List originalSplits, int desiredNumSplits, @@ -212,7 +214,7 @@ public List getGroupedSplits(Configuration conf, locations = emptyLocations; allSplitsHaveLocalhost = false; } - for (String location : locations ) { + for (String location : locations) { if (location == null) { location = emptyLocation; allSplitsHaveLocalhost = false; @@ -224,7 +226,7 @@ public List getGroupedSplits(Configuration conf, } } - if (! (configNumSplits > 0 || + if (!(configNumSplits > 0 || originalSplits.size() == 0)) { // numSplits has not been overridden by config // numSplits has been set at runtime @@ -232,8 +234,8 @@ public List getGroupedSplits(Configuration conf, // desired splits is less than number of splits generated // Do sanity checks - int splitCount = desiredNumSplits>0?desiredNumSplits:originalSplits.size(); - long lengthPerGroup = totalLength/splitCount; + int splitCount = desiredNumSplits > 0 ? desiredNumSplits : originalSplits.size(); + long lengthPerGroup = totalLength / splitCount; long maxLengthPerGroup = conf.getLong( TEZ_GROUPING_SPLIT_MAX_SIZE, @@ -242,14 +244,14 @@ public List getGroupedSplits(Configuration conf, TEZ_GROUPING_SPLIT_MIN_SIZE, TEZ_GROUPING_SPLIT_MIN_SIZE_DEFAULT); if (maxLengthPerGroup < minLengthPerGroup || - minLengthPerGroup <=0) { + minLengthPerGroup <= 0) { throw new TezUncheckedException( "Invalid max/min group lengths. Required min>0, max>=min. " + " max: " + maxLengthPerGroup + " min: " + minLengthPerGroup); } if (lengthPerGroup > maxLengthPerGroup) { // splits too big to work. Need to override with max size. - int newDesiredNumSplits = (int)(totalLength/maxLengthPerGroup) + 1; + int newDesiredNumSplits = (int) (totalLength / maxLengthPerGroup) + 1; LOG.info("Desired splits: " + desiredNumSplits + " too small. " + " Desired splitLength: " + lengthPerGroup + " Max splitLength: " + maxLengthPerGroup + @@ -260,7 +262,7 @@ public List getGroupedSplits(Configuration conf, desiredNumSplits = newDesiredNumSplits; } else if (lengthPerGroup < minLengthPerGroup) { // splits too small to work. Need to override with size. - int newDesiredNumSplits = (int)(totalLength/minLengthPerGroup) + 1; + int newDesiredNumSplits = (int) (totalLength / minLengthPerGroup) + 1; /** * This is a workaround for systems like S3 that pass the same * fake hostname for all splits. @@ -297,14 +299,14 @@ public List getGroupedSplits(Configuration conf, return groupedSplits; } - long lengthPerGroup = totalLength/desiredNumSplits; + long lengthPerGroup = totalLength / desiredNumSplits; int numNodeLocations = distinctLocations.size(); - int numSplitsPerLocation = originalSplits.size()/numNodeLocations; - int numSplitsInGroup = originalSplits.size()/desiredNumSplits; + int numSplitsPerLocation = originalSplits.size() / numNodeLocations; + int numSplitsInGroup = originalSplits.size() / desiredNumSplits; // allocation loop here so that we have a good initial size for the lists for (String location : distinctLocations.keySet()) { - distinctLocations.put(location, new LocationHolder(numSplitsPerLocation+1)); + distinctLocations.put(location, new LocationHolder(numSplitsPerLocation + 1)); } Set locSet = new HashSet(); @@ -381,7 +383,7 @@ public List getGroupedSplits(Configuration conf, groupNumSplits++; holder.incrementHeadIndex(); splitContainer = holder.getUnprocessedHeadSplit(); - } while(splitContainer != null + } while (splitContainer != null && (!groupByLength || (groupLength + estimator.getEstimatedSize(splitContainer) <= lengthPerGroup)) && (!groupByCount || @@ -389,8 +391,8 @@ public List getGroupedSplits(Configuration conf, if (holder.isEmpty() && !allowSmallGroups - && (!groupByLength || groupLength < lengthPerGroup/2) - && (!groupByCount || groupNumSplits < numSplitsInGroup/2)) { + && (!groupByLength || groupLength < lengthPerGroup / 2) + && (!groupByCount || groupNumSplits < numSplitsInGroup / 2)) { // group too small, reset it holder.headIndex = oldHeadIndex; continue; @@ -419,7 +421,7 @@ public List getGroupedSplits(Configuration conf, new GroupedSplitContainer(group.size(), wrappedInputFormatName, groupLocation, // pass rack local hint directly to AM - ((doingRackLocal && location != emptyLocation)?location:null)); + ((doingRackLocal && location != emptyLocation) ? location : null)); for (SplitContainer groupedSplitContainer : group) { groupedSplit.addSplit(groupedSplitContainer); Preconditions.checkState(groupedSplitContainer.isProcessed() == false, @@ -503,7 +505,7 @@ public List getGroupedSplits(Configuration conf, if (locations == null || locations.length == 0) { locations = emptyLocations; } - for (String location : locations ) { + for (String location : locations) { if (location == null) { location = emptyLocation; } @@ -521,8 +523,8 @@ public List getGroupedSplits(Configuration conf, TEZ_GROUPING_RACK_SPLIT_SIZE_REDUCTION, TEZ_GROUPING_RACK_SPLIT_SIZE_REDUCTION_DEFAULT); if (rackSplitReduction > 0) { - long newLengthPerGroup = (long)(lengthPerGroup*rackSplitReduction); - int newNumSplitsInGroup = (int) (numSplitsInGroup*rackSplitReduction); + long newLengthPerGroup = (long) (lengthPerGroup * rackSplitReduction); + int newNumSplitsInGroup = (int) (numSplitsInGroup * rackSplitReduction); if (newLengthPerGroup > 0) { lengthPerGroup = newLengthPerGroup; } @@ -542,7 +544,7 @@ public List getGroupedSplits(Configuration conf, continue; } - if (!allowSmallGroups && numFullGroupsCreated <= numNodeLocations/10) { + if (!allowSmallGroups && numFullGroupsCreated <= numNodeLocations / 10) { // a few nodes have a lot of data or data is thinly spread across nodes // so allow small groups now allowSmallGroups = true; @@ -596,12 +598,11 @@ private String[] cleanupLocations(String[] locations) { /** * Builder that can be used to configure grouping in Tez * - * @param conf - * {@link Configuration} This will be modified in place. If - * configuration values may be changed at runtime via a config file - * then pass in a {@link Configuration} that is initialized from a - * config file. The parameters that are not overridden in code will - * be derived from the Configuration object. + * @param conf {@link Configuration} This will be modified in place. If + * configuration values may be changed at runtime via a config file + * then pass in a {@link Configuration} that is initialized from a + * config file. The parameters that are not overridden in code will + * be derived from the Configuration object. * @return {@link org.apache.tez.mapreduce.grouper.TezSplitGrouper.TezMRSplitsGrouperConfigBuilder} */ public static TezMRSplitsGrouperConfigBuilder newConfigBuilder(Configuration conf) { diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java index 098057ba17..b34922fce4 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.mapreduce.hadoop; @@ -29,8 +29,6 @@ public class DeprecatedKeys { - - // This could be done via deprecation. /** * Keys used by the DAG - mainly the AM. @@ -43,8 +41,6 @@ public class DeprecatedKeys { private static Map mrParamToTezRuntimeParamMap = new HashMap(); - - static { populateMRToTezRuntimeParamMap(); populateMRToDagParamMap(); @@ -55,7 +51,7 @@ private static void populateMRToDagParamMap() { // TODO Default value handling. mrParamToDAGParamMap.put(MRJobConfig.MR_AM_TASK_LISTENER_THREAD_COUNT, TezConfiguration.TEZ_AM_TASK_LISTENER_THREAD_COUNT); - + mrParamToDAGParamMap.put(MRJobConfig.MAX_TASK_FAILURES_PER_TRACKER, TezConfiguration.TEZ_AM_MAX_TASK_FAILURES_PER_NODE); mrParamToDAGParamMap.put(MRJobConfig.MR_AM_JOB_NODE_BLACKLISTING_ENABLE, @@ -65,7 +61,7 @@ private static void populateMRToDagParamMap() { TezConfiguration.TEZ_AM_NODE_BLACKLISTING_IGNORE_THRESHOLD); mrParamToDAGParamMap.put(MRJobConfig.QUEUE_NAME, - TezConfiguration.TEZ_QUEUE_NAME); + TezConfiguration.TEZ_QUEUE_NAME); // Counter replacement will work in this manner, as long as TezCounters // extends MRCounters and is used directly by the Mapper/Reducer. @@ -74,13 +70,13 @@ private static void populateMRToDagParamMap() { // Framework counters, like FILESYSTEM will likely be incompatible since // they enum key belongs to a different package. mrParamToDAGParamMap.put(MRJobConfig.COUNTERS_MAX_KEY, - TezConfiguration.TEZ_COUNTERS_MAX); + TezConfiguration.TEZ_COUNTERS_MAX); mrParamToDAGParamMap.put(MRJobConfig.COUNTER_GROUPS_MAX_KEY, TezConfiguration.TEZ_COUNTERS_MAX_GROUPS); mrParamToDAGParamMap.put(MRJobConfig.COUNTER_NAME_MAX_KEY, TezConfiguration.TEZ_COUNTERS_COUNTER_NAME_MAX_LENGTH); mrParamToDAGParamMap.put(MRJobConfig.COUNTER_GROUP_NAME_MAX_KEY, - TezConfiguration.TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH); + TezConfiguration.TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH); mrParamToDAGParamMap.put(MRJobConfig.TASK_TIMEOUT, TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS); mrParamToDAGParamMap.put(MRJobConfig.JOB_TAGS, @@ -88,17 +84,17 @@ private static void populateMRToDagParamMap() { mrParamToDAGParamMap.put(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, TezConfiguration.TEZ_USER_CLASSPATH_FIRST); mrParamToDAGParamMap.put(MRJobConfig.JOB_NAMENODES, - TezConfiguration.TEZ_JOB_FS_SERVERS); + TezConfiguration.TEZ_JOB_FS_SERVERS); mrParamToDAGParamMap.put(MRJobConfig.JOB_NAMENODES_TOKEN_RENEWAL_EXCLUDE, - TezConfiguration.TEZ_JOB_FS_SERVERS_TOKEN_RENEWAL_EXCLUDE); + TezConfiguration.TEZ_JOB_FS_SERVERS_TOKEN_RENEWAL_EXCLUDE); } // TODO TEZAM4 Sometime, make sure this gets loaded by default. Instead of the current initialization in MRAppMaster, TezChild. // Maybe define in an TEZConfiguration / TEZ JobConf variant. - + public static void init() { } - + private static void populateMRToTezRuntimeParamMap() { registerMRToRuntimeKeyTranslation(MRConfig.MAPRED_IFILE_READAHEAD, TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD); @@ -108,49 +104,49 @@ private static void populateMRToTezRuntimeParamMap() { registerMRToRuntimeKeyTranslation(MRJobConfig.RECORDS_BEFORE_PROGRESS, TezRuntimeConfiguration.TEZ_RUNTIME_RECORDS_BEFORE_PROGRESS); registerMRToRuntimeKeyTranslation(MRJobConfig.IO_SORT_FACTOR, TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR); - + registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_SORT_SPILL_PERCENT, TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT); - + registerMRToRuntimeKeyTranslation(MRJobConfig.IO_SORT_MB, TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB); - + registerMRToRuntimeKeyTranslation(MRJobConfig.INDEX_CACHE_MEMORY_LIMIT, TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES); - + registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_COMBINE_MIN_SPILLS, TezRuntimeConfiguration.TEZ_RUNTIME_COMBINE_MIN_SPILLS); - + registerMRToRuntimeKeyTranslation(MRJobConfig.REDUCE_MEMORY_TOTAL_BYTES, Constants.TEZ_RUNTIME_TASK_MEMORY); - + registerMRToRuntimeKeyTranslation(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION); registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_PARALLEL_COPIES, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES); - + registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_FETCH_FAILURES, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT); - + registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_NOTIFY_READERROR, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR); - + registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_CONNECT_TIMEOUT, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_CONNECT_TIMEOUT); - + registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_READ_TIMEOUT, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT); - + registerMRToRuntimeKeyTranslation(MRConfig.SHUFFLE_SSL_ENABLED_KEY, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_SSL); - + registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT); - + registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT); - + registerMRToRuntimeKeyTranslation(MRJobConfig.SHUFFLE_MERGE_PERCENT, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT); - + registerMRToRuntimeKeyTranslation(MRJobConfig.REDUCE_MEMTOMEM_THRESHOLD, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS); - + registerMRToRuntimeKeyTranslation(MRJobConfig.REDUCE_MEMTOMEM_ENABLED, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM); - + registerMRToRuntimeKeyTranslation(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT, TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT); - + registerMRToRuntimeKeyTranslation("map.sort.class", TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS); - + registerMRToRuntimeKeyTranslation(MRJobConfig.GROUP_COMPARATOR_CLASS, TezRuntimeConfiguration.TEZ_RUNTIME_GROUP_COMPARATOR_CLASS); - + registerMRToRuntimeKeyTranslation(MRJobConfig.GROUP_COMPARATOR_CLASS, TezRuntimeConfiguration.TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS); - + registerMRToRuntimeKeyTranslation(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, TezConfiguration.TEZ_CREDENTIALS_PATH); registerMRToRuntimeKeyTranslation(MRJobConfig.KEY_COMPARATOR, TezRuntimeConfiguration.TEZ_RUNTIME_KEY_COMPARATOR_CLASS); @@ -175,12 +171,12 @@ private static void populateMRToTezRuntimeParamMap() { registerMRToRuntimeKeyTranslation(MRJobConfig.MINIMUM_ALLOWED_TASKS, TezConfiguration.TEZ_AM_MINIMUM_ALLOWED_SPECULATIVE_TASKS); } - + private static void addDeprecatedKeys() { } private static void registerMRToRuntimeKeyTranslation(String mrKey, - String tezKey) { + String tezKey) { mrParamToTezRuntimeParamMap.put(mrKey, tezKey); } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java index 2fddd00522..06813c4bb0 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/IDConverter.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.mapreduce.hadoop; @@ -36,14 +36,14 @@ public static JobID toMRJobId(TezDAGID dagId) { Long.toString(dagId.getApplicationId().getClusterTimestamp()), dagId.getApplicationId().getId()); } - + public static TaskID toMRTaskId(TezTaskID taskid) { return new TaskID( toMRJobId(taskid.getDAGID()), taskid.getVertexID().getId() == 0 ? TaskType.MAP : TaskType.REDUCE, taskid.getId()); } - + public static TaskID toMRTaskIdForOutput(TezTaskID taskid) { return org.apache.tez.mapreduce.hadoop.mapreduce.TaskAttemptContextImpl .createMockTaskAttemptIDFromTezTaskId(taskid, (taskid.getVertexID().getId() == 0)); @@ -55,7 +55,7 @@ public static TaskAttemptID toMRTaskAttemptId( toMRTaskId(taskAttemptId.getTaskID()), taskAttemptId.getId()); } - + // FIXME hardcoded assumption that one app is one dag public static TezDAGID fromMRJobId( org.apache.hadoop.mapreduce.JobID jobId) { @@ -65,11 +65,11 @@ public static TezDAGID fromMRJobId( // FIXME hack alert converting objects with hard coded id public static TezTaskID - fromMRTaskId(org.apache.hadoop.mapreduce.TaskID taskid) { + fromMRTaskId(org.apache.hadoop.mapreduce.TaskID taskid) { return TezTaskID.getInstance( TezVertexID.getInstance(fromMRJobId(taskid.getJobID()), - (taskid.getTaskType() == TaskType.MAP ? 0 : 1) - ), + (taskid.getTaskType() == TaskType.MAP ? 0 : 1) + ), taskid.getId()); } @@ -79,5 +79,4 @@ public static TezTaskAttemptID fromMRTaskAttemptId( fromMRTaskId(taskAttemptId.getTaskID()), taskAttemptId.getId()); } - } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/InputSplitInfo.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/InputSplitInfo.java index dfc152dde1..48921194f9 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/InputSplitInfo.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/InputSplitInfo.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,10 +31,10 @@ /** * Provides information about input splits.

    - * + * * The get*Path methods are only applicable when generating splits to disk. The * getSplitsProto method is only applicable when generating splits to memory. - * + * */ @Private @Unstable @@ -43,6 +43,7 @@ public interface InputSplitInfo { public enum Type { DISK, MEM } + /** * Get the TaskLocationHints for each task */ @@ -62,18 +63,18 @@ public enum Type { * Get the splits proto */ public abstract MRSplitsProto getSplitsProto(); - + /** * Get the number of splits that were generated. Same as number of tasks that * should be run for the vertex processing these set of splits. */ public abstract int getNumTasks(); - + /** * Get the {@link Type} of the InputSplitInfo */ public abstract Type getType(); - + /** * Get {@link Credentials} which may be required to access the splits. * @return {@link Credentials} which may be required to access the splits. diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/InputSplitInfoDisk.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/InputSplitInfoDisk.java index 5cc0d8edc5..5224c8ca1f 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/InputSplitInfoDisk.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/InputSplitInfoDisk.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,7 +36,7 @@ * the vertex and the taskLocationHints define the possible nodes on which the * tasks should be run based on the location of the splits that will be * processed by each task. - * + * * The getSplitsProto method is not supported by this implementation. */ public class InputSplitInfoDisk implements InputSplitInfo { @@ -52,7 +52,7 @@ public class InputSplitInfoDisk implements InputSplitInfo { private final Credentials credentials; public InputSplitInfoDisk(Path splitsFile, Path splitsMetaInfoFile, int numTasks, - List taskLocationHints, Credentials credentials) { + List taskLocationHints, Credentials credentials) { this.splitsFile = splitsFile; this.splitsMetaInfoFile = splitsMetaInfoFile; this.taskLocationHints = taskLocationHints; @@ -102,7 +102,7 @@ public MRSplitsProto getSplitsProto() { throw new UnsupportedOperationException("Not supported for Type: " + getType()); } - + @Override public Credentials getCredentials() { return this.credentials; @@ -125,5 +125,4 @@ public org.apache.hadoop.mapred.InputSplit[] getOldFormatSplits() { throw new UnsupportedOperationException("Not supported for Type: " + getType()); } - } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/InputSplitInfoMem.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/InputSplitInfoMem.java index 5e1207b7b5..3573fe7a71 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/InputSplitInfoMem.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/InputSplitInfoMem.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -32,28 +32,28 @@ /** * Represents InputSplitInfo for splits generated to memory.

    - * + * * Since splits are generated in memory, the getSplitsMetaInfoFile and * getSplitsFile are not supported. - * + * */ public class InputSplitInfoMem implements InputSplitInfo { -// private final MRSplitsProto splitsProto; + // private final MRSplitsProto splitsProto; private final boolean isNewSplit; private final int numTasks; private final Credentials credentials; private final Configuration conf; private final List taskLocationHints; - + private org.apache.hadoop.mapreduce.InputSplit[] newFormatSplits; private org.apache.hadoop.mapred.InputSplit[] oldFormatSplits; // TaskLocationHints accepted as a parameter since InputSplits don't have rack // info, and it can't always be derived. public InputSplitInfoMem(org.apache.hadoop.mapreduce.InputSplit[] newSplits, - List taskLocationHints, int numTasks, Credentials credentials, - Configuration conf) { + List taskLocationHints, int numTasks, Credentials credentials, + Configuration conf) { this.isNewSplit = true; this.newFormatSplits = newSplits; this.taskLocationHints = taskLocationHints; @@ -65,8 +65,8 @@ public InputSplitInfoMem(org.apache.hadoop.mapreduce.InputSplit[] newSplits, // TaskLocationHints accepted as a parameter since InputSplits don't have rack // info, and it can't always be derived. public InputSplitInfoMem(org.apache.hadoop.mapred.InputSplit[] oldSplits, - List taskLocationHints, int numTasks, Credentials credentials, - Configuration conf) { + List taskLocationHints, int numTasks, Credentials credentials, + Configuration conf) { this.isNewSplit = false; this.oldFormatSplits = oldSplits; this.taskLocationHints = taskLocationHints; @@ -128,7 +128,7 @@ public Credentials getCredentials() { @Override public boolean holdsNewFormatSplits() { - return this.isNewSplit; + return this.isNewSplit; } @Override diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRConfig.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRConfig.java index c93c675a79..1997d9b414 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRConfig.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRConfig.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -33,55 +33,55 @@ public interface MRConfig { public static final String LOCAL_DIR = "mapreduce.cluster.local.dir"; public static final String MAPMEMORY_MB = "mapreduce.cluster.mapmemory.mb"; public static final String REDUCEMEMORY_MB = - "mapreduce.cluster.reducememory.mb"; + "mapreduce.cluster.reducememory.mb"; public static final String MR_ACLS_ENABLED = "mapreduce.cluster.acls.enabled"; public static final String MR_ADMINS = - "mapreduce.cluster.administrators"; + "mapreduce.cluster.administrators"; @Deprecated public static final String MR_SUPERGROUP = - "mapreduce.cluster.permissions.supergroup"; + "mapreduce.cluster.permissions.supergroup"; //Delegation token related keys - public static final String DELEGATION_KEY_UPDATE_INTERVAL_KEY = - "mapreduce.cluster.delegation.key.update-interval"; - public static final long DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT = - 24*60*60*1000; // 1 day - public static final String DELEGATION_TOKEN_RENEW_INTERVAL_KEY = - "mapreduce.cluster.delegation.token.renew-interval"; - public static final long DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = - 24*60*60*1000; // 1 day - public static final String DELEGATION_TOKEN_MAX_LIFETIME_KEY = - "mapreduce.cluster.delegation.token.max-lifetime"; - public static final long DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = - 7*24*60*60*1000; // 7 days + public static final String DELEGATION_KEY_UPDATE_INTERVAL_KEY = + "mapreduce.cluster.delegation.key.update-interval"; + public static final long DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT = + 24 * 60 * 60 * 1000; // 1 day + public static final String DELEGATION_TOKEN_RENEW_INTERVAL_KEY = + "mapreduce.cluster.delegation.token.renew-interval"; + public static final long DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = + 24 * 60 * 60 * 1000; // 1 day + public static final String DELEGATION_TOKEN_MAX_LIFETIME_KEY = + "mapreduce.cluster.delegation.token.max-lifetime"; + public static final long DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = + 7 * 24 * 60 * 60 * 1000; // 7 days public static final String RESOURCE_CALCULATOR_PROCESS_TREE = - "mapreduce.job.process-tree.class"; + "mapreduce.job.process-tree.class"; public static final String STATIC_RESOLUTIONS = - "mapreduce.job.net.static.resolutions"; + "mapreduce.job.net.static.resolutions"; - public static final String MASTER_ADDRESS = "mapreduce.jobtracker.address"; + public static final String MASTER_ADDRESS = "mapreduce.jobtracker.address"; public static final String MASTER_USER_NAME = - "mapreduce.jobtracker.kerberos.principal"; + "mapreduce.jobtracker.kerberos.principal"; - public static final String FRAMEWORK_NAME = "mapreduce.framework.name"; - public static final String CLASSIC_FRAMEWORK_NAME = "classic"; - public static final String YARN_TEZ_FRAMEWORK_NAME = "yarn-tez"; + public static final String FRAMEWORK_NAME = "mapreduce.framework.name"; + public static final String CLASSIC_FRAMEWORK_NAME = "classic"; + public static final String YARN_TEZ_FRAMEWORK_NAME = "yarn-tez"; public static final String LOCAL_FRAMEWORK_NAME = "local"; public static final String TASK_LOCAL_OUTPUT_CLASS = - "mapreduce.task.local.output.class"; + "mapreduce.task.local.output.class"; public static final String PROGRESS_STATUS_LEN_LIMIT_KEY = - "mapreduce.task.max.status.length"; + "mapreduce.task.max.status.length"; public static final int PROGRESS_STATUS_LEN_LIMIT_DEFAULT = 512; public static final int MAX_BLOCK_LOCATIONS_DEFAULT = 10; public static final String MAX_BLOCK_LOCATIONS_KEY = - "mapreduce.job.max.split.locations"; + "mapreduce.job.max.split.locations"; public static final String SHUFFLE_SSL_ENABLED_KEY = - "mapreduce.shuffle.ssl.enabled"; + "mapreduce.shuffle.ssl.enabled"; public static final boolean SHUFFLE_SSL_ENABLED_DEFAULT = false; @@ -89,7 +89,7 @@ public interface MRConfig { * Configuration key to enable/disable IFile readahead. */ public static final String MAPRED_IFILE_READAHEAD = - "mapreduce.ifile.readahead"; + "mapreduce.ifile.readahead"; public static final boolean DEFAULT_MAPRED_IFILE_READAHEAD = true; @@ -97,10 +97,10 @@ public interface MRConfig { * Configuration key to set the IFile readahead length in bytes. */ public static final String MAPRED_IFILE_READAHEAD_BYTES = - "mapreduce.ifile.readahead.bytes"; + "mapreduce.ifile.readahead.bytes"; public static final int DEFAULT_MAPRED_IFILE_READAHEAD_BYTES = - 4 * 1024 * 1024; + 4 * 1024 * 1024; public static final String IS_MAP_PROCESSOR = "tez.mapreduce.is_map_processor"; diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java index 1d06b7aeaf..2cd20bebc8 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -38,7 +38,6 @@ import org.apache.tez.mapreduce.partition.MRPartitioner; import org.apache.tez.runtime.library.api.TezRuntimeConfiguration; - /** * This class contains helper methods for frameworks which migrate from MapReduce to Tez, and need * to continue to work with existing MapReduce configurations. @@ -49,7 +48,6 @@ public class MRHelpers { private static final Logger LOG = LoggerFactory.getLogger(MRHelpers.class); - /** * Translate MapReduce configuration keys to the equivalent Tez keys in the provided * configuration. The translation is done in place.

    @@ -75,7 +73,6 @@ public static void translateMRConfToTez(Configuration conf, boolean preferTez) { convertVertexConfToTez(conf, preferTez); } - /** * Update the provided configuration to use the new API (mapreduce) or the old API (mapred) based * on the configured InputFormat, OutputFormat, Partitioner etc. Also ensures that keys not @@ -205,12 +202,12 @@ private static String getChildLogLevel(Configuration conf, boolean isMap) { return conf.get( MRJobConfig.MAP_LOG_LEVEL, MRJobConfig.DEFAULT_LOG_LEVEL - ); + ); } else { return conf.get( MRJobConfig.REDUCE_LOG_LEVEL, MRJobConfig.DEFAULT_LOG_LEVEL - ); + ); } } @@ -222,7 +219,7 @@ private static void ensureNotSet(Configuration conf, String attr, String msg) } private static String getLog4jCmdLineProperties(Configuration conf, - boolean isMap) { + boolean isMap) { Vector logProps = new Vector(4); TezUtils.addLog4jSystemProperties(getChildLogLevel(conf, isMap), logProps); StringBuilder sb = new StringBuilder(); @@ -328,8 +325,8 @@ public static String getJavaOptsForMRReducer(Configuration conf) { * running Map tasks */ public static Resource getResourceForMRMapper(Configuration conf) { - JobConf jobConf = conf instanceof JobConf ? (JobConf)conf : new JobConf(conf); - return Resource.newInstance((int)jobConf.getMemoryForMapTask(), + JobConf jobConf = conf instanceof JobConf ? (JobConf) conf : new JobConf(conf); + return Resource.newInstance((int) jobConf.getMemoryForMapTask(), jobConf.getInt(MRJobConfig.MAP_CPU_VCORES, MRJobConfig.DEFAULT_MAP_CPU_VCORES)); } @@ -350,8 +347,8 @@ public static Resource getResourceForMRMapper(Configuration conf) { * running Reduce tasks */ public static Resource getResourceForMRReducer(Configuration conf) { - JobConf jobConf = conf instanceof JobConf ? (JobConf)conf : new JobConf(conf); - return Resource.newInstance((int)jobConf.getMemoryForReduceTask(), + JobConf jobConf = conf instanceof JobConf ? (JobConf) conf : new JobConf(conf); + return Resource.newInstance((int) jobConf.getMemoryForReduceTask(), conf.getInt(MRJobConfig.REDUCE_CPU_VCORES, MRJobConfig.DEFAULT_REDUCE_CPU_VCORES)); } @@ -375,8 +372,8 @@ public static void updateEnvBasedOnMRTaskEnv(Configuration conf, // Add the env variables passed by the admin TezYARNUtils.appendToEnvFromInputString(environment, conf.get( - MRJobConfig.MAPRED_ADMIN_USER_ENV, - MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV), + MRJobConfig.MAPRED_ADMIN_USER_ENV, + MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV), File.pathSeparator); // Add the env variables passed by the user @@ -402,5 +399,4 @@ public static void updateEnvBasedOnMRAMEnv(Configuration conf, Map + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -112,7 +112,7 @@ public static DataSourceDescriptor configureMRInputWithLegacySplitGeneration(Con inputSplitInfo = generateInputSplits(conf, splitsDir); InputDescriptor inputDescriptor = InputDescriptor.create(useLegacyInput ? MRInputLegacy.class - .getName() : MRInput.class.getName()) + .getName() : MRInput.class.getName()) .setUserPayload(createMRInputPayload(conf, null, false, true)); Map additionalLocalResources = new HashMap(); updateLocalResourcesForInputSplits(conf, inputSplitInfo, @@ -132,7 +132,6 @@ public static DataSourceDescriptor configureMRInputWithLegacySplitGeneration(Con } } - /** * Parse the payload used by MRInputPayload * @@ -316,7 +315,7 @@ public static InputSplitInfoMem generateInputSplitsToMem(Configuration conf, */ @InterfaceStability.Unstable public static InputSplitInfoMem generateInputSplitsToMem(Configuration conf, - boolean groupSplits, boolean sortSplits, int targetTasks) + boolean groupSplits, boolean sortSplits, int targetTasks) throws IOException, ClassNotFoundException, InterruptedException { InputSplitInfoMem splitInfoMem = null; @@ -410,14 +409,13 @@ public TaskLocationHint apply(org.apache.hadoop.mapred.InputSplit input) { return Lists.newArrayList(iterable); } - @SuppressWarnings({ "rawtypes", "unchecked" }) + @SuppressWarnings({"rawtypes", "unchecked"}) private static org.apache.hadoop.mapreduce.InputSplit[] generateNewSplits( JobContext jobContext, boolean groupSplits, boolean sortSplits, int numTasks) throws ClassNotFoundException, IOException, InterruptedException { Configuration conf = jobContext.getConfiguration(); - // This is the real input format. org.apache.hadoop.mapreduce.InputFormat inputFormat = null; try { @@ -456,7 +454,7 @@ private static org.apache.hadoop.mapreduce.InputSplit[] generateNewSplits( return splits; } - @SuppressWarnings({ "rawtypes", "unchecked" }) + @SuppressWarnings({"rawtypes", "unchecked"}) private static org.apache.hadoop.mapred.InputSplit[] generateOldSplits( JobConf jobConf, boolean groupSplits, boolean sortSplits, int numTasks) throws IOException { @@ -740,9 +738,9 @@ protected static UserPayload createMRInputPayload(Configuration conf, */ @InterfaceAudience.Private protected static UserPayload createMRInputPayload(Configuration conf, - MRRuntimeProtos.MRSplitsProto mrSplitsProto, boolean isGrouped, - boolean isSorted) throws - IOException { + MRRuntimeProtos.MRSplitsProto mrSplitsProto, boolean isGrouped, + boolean isSorted) throws + IOException { Preconditions .checkArgument(conf != null, "Configuration must be specified"); @@ -751,8 +749,8 @@ protected static UserPayload createMRInputPayload(Configuration conf, } private static UserPayload createMRInputPayload(ByteString bytes, - MRRuntimeProtos.MRSplitsProto mrSplitsProto, - boolean isGrouped, boolean isSorted) throws IOException { + MRRuntimeProtos.MRSplitsProto mrSplitsProto, + boolean isGrouped, boolean isSorted) throws IOException { MRRuntimeProtos.MRInputUserPayloadProto.Builder userPayloadBuilder = MRRuntimeProtos.MRInputUserPayloadProto .newBuilder(); @@ -766,7 +764,6 @@ private static UserPayload createMRInputPayload(ByteString bytes, toByteString().asReadOnlyByteBuffer()); } - private static String getStringProperty(Configuration conf, String propertyName) { Objects.requireNonNull(conf, "Configuration must be provided"); Objects.requireNonNull(propertyName, "Property name must be provided"); @@ -926,5 +923,4 @@ public static String getUniqueIdentifier(Configuration conf) { public static int getDagAttemptNumber(Configuration conf) { return getIntProperty(conf, MRInput.TEZ_MAPREDUCE_DAG_ATTEMPT_NUMBER); } - } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java index e162460773..156c157e7d 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -27,13 +27,13 @@ public interface MRJobConfig { @Private static final String MR_TEZ_PREFIX = "mapreduce.tez."; - + // Put all of the attribute names in here so that Job and JobContext are // consistent. public static final String INPUT_FORMAT_CLASS_ATTR = "mapreduce.job.inputformat.class"; - + public static final String NEW_API_MAPPER_CONFIG = "mapred.mapper.new-api"; - + public static final String NEW_API_REDUCER_CONFIG = "mapred.reducer.new-api"; public static final String MAP_CLASS_ATTR = "mapreduce.job.map.class"; @@ -69,7 +69,7 @@ public interface MRJobConfig { public static final String JVM_NUMTASKS_TORUN = "mapreduce.job.jvm.numtasks"; public static final String SPLIT_FILE = "mapreduce.job.splitfile"; - + public static final String SPLIT_METAINFO_MAXSIZE = "mapreduce.job.split.metainfo.maxsize"; public static final long DEFAULT_SPLIT_METAINFO_MAXSIZE = 10000000L; @@ -182,13 +182,13 @@ public interface MRJobConfig { public static final String NUM_REDUCE_PROFILES = "mapreduce.task.profile.reduces"; public static final String TASK_MAP_PROFILE_PARAMS = "mapreduce.task.profile.map.params"; - + public static final String TASK_REDUCE_PROFILE_PARAMS = "mapreduce.task.profile.reduce.params"; - + public static final String TASK_TIMEOUT = "mapreduce.task.timeout"; public static final String TASK_TIMEOUT_CHECK_INTERVAL_MS = "mapreduce.task.timeout.check-interval-ms"; - + public static final String TASK_ID = "mapreduce.task.id"; public static final String TASK_OUTPUT_DIR = "mapreduce.task.output.dir"; @@ -264,7 +264,7 @@ public interface MRJobConfig { public static final String SHUFFLE_INPUT_BUFFER_PERCENT = "mapreduce.reduce.shuffle.input.buffer.percent"; public static final String SHUFFLE_MEMORY_LIMIT_PERCENT - = "mapreduce.reduce.shuffle.memory.limit.percent"; + = "mapreduce.reduce.shuffle.memory.limit.percent"; public static final String SHUFFLE_MERGE_PERCENT = "mapreduce.reduce.shuffle.merge.percent"; @@ -315,17 +315,17 @@ public interface MRJobConfig { public static final String JOB_ACL_MODIFY_JOB = "mapreduce.job.acl-modify-job"; public static final String DEFAULT_JOB_ACL_MODIFY_JOB = " "; - + /* config for tracking the local file where all the credentials for the job * credentials. */ - public static final String MAPREDUCE_JOB_CREDENTIALS_BINARY = + public static final String MAPREDUCE_JOB_CREDENTIALS_BINARY = "mapreduce.job.credentials.binary"; public static final String JOB_SUBMITHOST = - "mapreduce.job.submithostname"; + "mapreduce.job.submithostname"; public static final String JOB_SUBMITHOSTADDR = - "mapreduce.job.submithostaddress"; + "mapreduce.job.submithostaddress"; public static final String COUNTERS_MAX_KEY = "mapreduce.job.counters.max"; public static final int COUNTERS_MAX_DEFAULT = 120; @@ -339,13 +339,13 @@ public interface MRJobConfig { public static final String COUNTER_GROUPS_MAX_KEY = "mapreduce.job.counters.groups.max"; public static final int COUNTER_GROUPS_MAX_DEFAULT = 50; public static final String JOB_UBERTASK_ENABLE = - "mapreduce.job.ubertask.enable"; + "mapreduce.job.ubertask.enable"; public static final String JOB_UBERTASK_MAXMAPS = - "mapreduce.job.ubertask.maxmaps"; + "mapreduce.job.ubertask.maxmaps"; public static final String JOB_UBERTASK_MAXREDUCES = - "mapreduce.job.ubertask.maxreduces"; + "mapreduce.job.ubertask.maxreduces"; public static final String JOB_UBERTASK_MAXBYTES = - "mapreduce.job.ubertask.maxbytes"; + "mapreduce.job.ubertask.maxbytes"; public static final String MR_PREFIX = "yarn.app.mapreduce."; @@ -354,51 +354,51 @@ public interface MRJobConfig { /** The number of client retires to the AM - before reconnecting to the RM * to fetch Application State. */ - public static final String MR_CLIENT_TO_AM_IPC_MAX_RETRIES = - MR_PREFIX + "client-am.ipc.max-retries"; + public static final String MR_CLIENT_TO_AM_IPC_MAX_RETRIES = + MR_PREFIX + "client-am.ipc.max-retries"; public static final int DEFAULT_MR_CLIENT_TO_AM_IPC_MAX_RETRIES = 3; - + /** * The number of client retries to the RM/HS/AM before throwing exception. */ - public static final String MR_CLIENT_MAX_RETRIES = - MR_PREFIX + "client.max-retries"; + public static final String MR_CLIENT_MAX_RETRIES = + MR_PREFIX + "client.max-retries"; public static final int DEFAULT_MR_CLIENT_MAX_RETRIES = 3; - + /** The staging directory for map reduce.*/ - public static final String MR_AM_STAGING_DIR = - MR_AM_PREFIX+"staging-dir"; - public static final String DEFAULT_MR_AM_STAGING_DIR = - "/tmp/hadoop-yarn/staging"; + public static final String MR_AM_STAGING_DIR = + MR_AM_PREFIX + "staging-dir"; + public static final String DEFAULT_MR_AM_STAGING_DIR = + "/tmp/hadoop-yarn/staging"; /** The amount of memory the MR app master needs.*/ public static final String MR_AM_VMEM_MB = - MR_AM_PREFIX+"resource.mb"; + MR_AM_PREFIX + "resource.mb"; public static final int DEFAULT_MR_AM_VMEM_MB = 1536; /** The number of virtual cores the MR app master needs.*/ public static final String MR_AM_CPU_VCORES = - MR_AM_PREFIX+"resource.cpu-vcores"; + MR_AM_PREFIX + "resource.cpu-vcores"; public static final int DEFAULT_MR_AM_CPU_VCORES = 1; /** Command line arguments passed to the MR app master.*/ public static final String MR_AM_COMMAND_OPTS = - MR_AM_PREFIX+"command-opts"; + MR_AM_PREFIX + "command-opts"; public static final String DEFAULT_MR_AM_COMMAND_OPTS = "-Xmx1024m"; /** Admin command opts passed to the MR app master.*/ public static final String MR_AM_ADMIN_COMMAND_OPTS = - MR_AM_PREFIX+"admin-command-opts"; + MR_AM_PREFIX + "admin-command-opts"; public static final String DEFAULT_MR_AM_ADMIN_COMMAND_OPTS = ""; /** Root Logging level passed to the MR app master.*/ - public static final String MR_AM_LOG_LEVEL = - MR_AM_PREFIX+"log.level"; + public static final String MR_AM_LOG_LEVEL = + MR_AM_PREFIX + "log.level"; public static final String DEFAULT_MR_AM_LOG_LEVEL = "INFO"; /**The number of splits when reporting progress in MR*/ - public static final String MR_AM_NUM_PROGRESS_SPLITS = - MR_AM_PREFIX+"num-progress-splits"; + public static final String MR_AM_NUM_PROGRESS_SPLITS = + MR_AM_PREFIX + "num-progress-splits"; public static final int DEFAULT_MR_AM_NUM_PROGRESS_SPLITS = 12; /** @@ -406,88 +406,88 @@ public interface MRJobConfig { * master. Expect level config, you shouldn't be needing it in most cases. */ public static final String MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT = - MR_AM_PREFIX+"containerlauncher.thread-count-limit"; + MR_AM_PREFIX + "containerlauncher.thread-count-limit"; - public static final int DEFAULT_MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT = + public static final int DEFAULT_MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT = 500; /** Number of threads to handle job client RPC requests.*/ public static final String MR_AM_JOB_CLIENT_THREAD_COUNT = - MR_AM_PREFIX + "job.client.thread-count"; + MR_AM_PREFIX + "job.client.thread-count"; public static final int DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT = 1; - /** + /** * Range of ports that the MapReduce AM can use when binding. Leave blank * if you want all possible ports. */ - public static final String MR_AM_JOB_CLIENT_PORT_RANGE = - MR_AM_PREFIX + "job.client.port-range"; - + public static final String MR_AM_JOB_CLIENT_PORT_RANGE = + MR_AM_PREFIX + "job.client.port-range"; + /** Enable blacklisting of nodes in the job.*/ - public static final String MR_AM_JOB_NODE_BLACKLISTING_ENABLE = - MR_AM_PREFIX + "job.node-blacklisting.enable"; + public static final String MR_AM_JOB_NODE_BLACKLISTING_ENABLE = + MR_AM_PREFIX + "job.node-blacklisting.enable"; /** Ignore blacklisting if a certain percentage of nodes have been blacklisted */ public static final String MR_AM_IGNORE_BLACKLISTING_BLACKLISTED_NODE_PERECENT = MR_AM_PREFIX + "job.node-blacklisting.ignore-threshold-node-percent"; public static final int DEFAULT_MR_AM_IGNORE_BLACKLISTING_BLACKLISTED_NODE_PERCENT = 33; - + /** Enable job recovery.*/ - public static final String MR_AM_JOB_RECOVERY_ENABLE = - MR_AM_PREFIX + "job.recovery.enable"; + public static final String MR_AM_JOB_RECOVERY_ENABLE = + MR_AM_PREFIX + "job.recovery.enable"; - /** + /** * Limit on the number of reducers that can be preempted to ensure that at * least one map task can run if it needs to. Percentage between 0.0 and 1.0 */ - public static final String MR_AM_JOB_REDUCE_PREEMPTION_LIMIT = - MR_AM_PREFIX + "job.reduce.preemption.limit"; + public static final String MR_AM_JOB_REDUCE_PREEMPTION_LIMIT = + MR_AM_PREFIX + "job.reduce.preemption.limit"; public static final float DEFAULT_MR_AM_JOB_REDUCE_PREEMPTION_LIMIT = 0.5f; - + /** AM ACL disabled. **/ - public static final String JOB_AM_ACCESS_DISABLED = - "mapreduce.job.am-access-disabled"; + public static final String JOB_AM_ACCESS_DISABLED = + "mapreduce.job.am-access-disabled"; public static final boolean DEFAULT_JOB_AM_ACCESS_DISABLED = false; /** * Limit reduces starting until a certain percentage of maps have finished. * Percentage between 0.0 and 1.0 */ - public static final String MR_AM_JOB_REDUCE_RAMPUP_UP_LIMIT = - MR_AM_PREFIX + "job.reduce.rampup.limit"; + public static final String MR_AM_JOB_REDUCE_RAMPUP_UP_LIMIT = + MR_AM_PREFIX + "job.reduce.rampup.limit"; public static final float DEFAULT_MR_AM_JOB_REDUCE_RAMP_UP_LIMIT = 0.5f; /** Class used to estimate task resource needs.*/ public static final String MR_AM_TASK_ESTIMATOR = - MR_AM_PREFIX + "job.task.estimator.class"; + MR_AM_PREFIX + "job.task.estimator.class"; /** The lambda value in the smoothing function of the task estimator.*/ public static final String MR_AM_TASK_ESTIMATOR_SMOOTH_LAMBDA_MS = - MR_AM_PREFIX - + "job.task.estimator.exponential.smooth.lambda-ms"; + MR_AM_PREFIX + + "job.task.estimator.exponential.smooth.lambda-ms"; - public static final long DEFAULT_MR_AM_TASK_ESTIMATOR_SMOOTH_LAMBDA_MS = - 1000L * 60; + public static final long DEFAULT_MR_AM_TASK_ESTIMATOR_SMOOTH_LAMBDA_MS = + 1000L * 60; /** true if the smoothing rate should be exponential.*/ public static final String MR_AM_TASK_ESTIMATOR_EXPONENTIAL_RATE_ENABLE = - MR_AM_PREFIX + "job.task.estimator.exponential.smooth.rate"; + MR_AM_PREFIX + "job.task.estimator.exponential.smooth.rate"; /** The number of threads used to handle task RPC calls.*/ public static final String MR_AM_TASK_LISTENER_THREAD_COUNT = - MR_AM_PREFIX + "job.task.listener.thread-count"; + MR_AM_PREFIX + "job.task.listener.thread-count"; public static final int DEFAULT_MR_AM_TASK_LISTENER_THREAD_COUNT = 30; /** How often the AM should schedule assigning tasks with allocated * containers.*/ public static final String MR_AM_SCHEDULER_INTERVAL = - MR_AM_PREFIX + "scheduler.interval-ms"; + MR_AM_PREFIX + "scheduler.interval-ms"; public static final long DEFAULT_MR_AM_SCHEDULER_INTERVAL = 1000l; /** How often the AM should send heartbeats to the RM.*/ public static final String MR_AM_TO_RM_HEARTBEAT_INTERVAL_MS = - MR_AM_PREFIX + "scheduler.heartbeat.interval-ms"; + MR_AM_PREFIX + "scheduler.heartbeat.interval-ms"; public static final int DEFAULT_MR_AM_TO_RM_HEARTBEAT_INTERVAL_MS = 1000; /** @@ -496,7 +496,7 @@ public interface MRJobConfig { * to contact the RM. */ public static final String MR_AM_TO_RM_WAIT_INTERVAL_MS = - MR_AM_PREFIX + "scheduler.connection.wait.interval-ms"; + MR_AM_PREFIX + "scheduler.connection.wait.interval-ms"; public static final int DEFAULT_MR_AM_TO_RM_WAIT_INTERVAL_MS = 360000; /** @@ -504,9 +504,9 @@ public interface MRJobConfig { * Set to false for multi-user clusters. This is an internal config that * is set by the MR framework and read by it too. */ - public static final String MR_AM_CREATE_JH_INTERMEDIATE_BASE_DIR = - MR_AM_PREFIX + "create-intermediate-jh-base-dir"; - + public static final String MR_AM_CREATE_JH_INTERMEDIATE_BASE_DIR = + MR_AM_PREFIX + "create-intermediate-jh-base-dir"; + public static final String MR_AM_HISTORY_MAX_UNFLUSHED_COMPLETE_EVENTS = MR_AM_PREFIX + "history.max-unflushed-events"; public static final int DEFAULT_MR_AM_HISTORY_MAX_UNFLUSHED_COMPLETE_EVENTS = @@ -526,10 +526,10 @@ public interface MRJobConfig { MR_AM_PREFIX + "history.use-batched-flush.queue-size.threshold"; public static final int DEFAULT_MR_AM_HISTORY_USE_BATCHED_FLUSH_QUEUE_SIZE_THRESHOLD = 50; - + public static final String MR_AM_ENV = MR_AM_PREFIX + "env"; - + public static final String MR_AM_ADMIN_USER_ENV = MR_AM_PREFIX + "admin.user.env"; @@ -586,10 +586,10 @@ public interface MRJobConfig { /** The log directory for the containers */ public static final String TASK_LOG_DIR = MR_PREFIX + "container.log.dir"; - + public static final String TASK_LOG_SIZE = MR_PREFIX + "container.log.filesize"; - - public static final String MAPREDUCE_V2_CHILD_CLASS = + + public static final String MAPREDUCE_V2_CHILD_CLASS = "org.apache.hadoop.mapred.YarnChild"; public static final String APPLICATION_ATTEMPT_ID = @@ -598,17 +598,17 @@ public interface MRJobConfig { /* * MR AM Service Authorization */ - public static final String - MR_AM_SECURITY_SERVICE_AUTHORIZATION_TASK_UMBILICAL = + public static final String + MR_AM_SECURITY_SERVICE_AUTHORIZATION_TASK_UMBILICAL = "security.job.task.protocol.acl"; - public static final String - MR_AM_SECURITY_SERVICE_AUTHORIZATION_CLIENT = + public static final String + MR_AM_SECURITY_SERVICE_AUTHORIZATION_CLIENT = "security.job.client.protocol.acl"; /** * CLASSPATH for all YARN MapReduce applications. */ - public static final String MAPREDUCE_APPLICATION_CLASSPATH = + public static final String MAPREDUCE_APPLICATION_CLASSPATH = "mapreduce.application.classpath"; /** @@ -628,7 +628,7 @@ public interface MRJobConfig { @InterfaceAudience.Private static final String HS_DELEGATION_TOKEN_REQUIRED = "mapreduce.history.server.delegationtoken.required"; - + public static final String MROUTPUT_FILE_NAME_PREFIX = MR_TEZ_PREFIX + "mroutput.file-name.prefix"; @@ -639,9 +639,9 @@ public interface MRJobConfig { public static final String MRR_INTERMEDIATE_STAGE_PREFIX = "mrr.intermediate.stage."; - + // TODO Move this over to Tez - public static final String MRR_VERTEX_PREFIX = + public static final String MRR_VERTEX_PREFIX = "mrr.vertex."; public static final String VERTEX_NAME = "mapreduce.task.vertex.name"; @@ -653,5 +653,4 @@ public interface MRJobConfig { public static final String MR_TEZ_INPUT_INITIALIZER_SERIALIZE_EVENT_PAYLOAD = MR_TEZ_PREFIX + "input.initializer.serialize.event.payload"; public static final boolean MR_TEZ_INPUT_INITIALIZER_SERIALIZE_EVENT_PAYLOAD_DEFAULT = true; - } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java index 3f5ad230cf..19c06207b8 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfToTezTranslator.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,13 +24,12 @@ import org.apache.tez.mapreduce.partition.MRPartitioner; import org.apache.tez.runtime.library.api.TezRuntimeConfiguration; - public class MultiStageMRConfToTezTranslator { /** * Given a single base MRR config, returns a list of complete stage * configurations. - * + * * @param conf * @return list of complete stage configurations given Conifiguration */ @@ -44,11 +43,11 @@ public static Configuration[] getStageConfs(Configuration conf) { // Read split info from HDFS conf.setBoolean(MRJobConfig.MR_TEZ_SPLITS_VIA_EVENTS, false); - + // Setup Tez partitioner class conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS, MRPartitioner.class.getName()); - + // Setup Tez Combiner class if required. // This would already have been set since the call is via JobClient boolean useNewApi = conf.getBoolean("mapred.mapper.new-api", false); @@ -71,7 +70,7 @@ public static Configuration[] getStageConfs(Configuration conf) { } else { confs[0] = nonItermediateConf; confs[numStages - 1] = new Configuration(nonItermediateConf); - confs[numStages -1].setBoolean(MRConfig.IS_MAP_PROCESSOR, false); + confs[numStages - 1].setBoolean(MRConfig.IS_MAP_PROCESSOR, false); } if (numStages > 2) { for (int i = 1; i < numStages - 1; i++) { @@ -82,4 +81,4 @@ public static Configuration[] getStageConfs(Configuration conf) { } return confs; } -} \ No newline at end of file +} diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfigUtil.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfigUtil.java index 13e0b860eb..92b01c7008 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfigUtil.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MultiStageMRConfigUtil.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -44,11 +44,11 @@ public static String getPropertyNameForIntermediateStage( return MRJobConfig.MRR_INTERMEDIATE_STAGE_PREFIX + intermediateStage + "." + originalPropertyName; } - - ////////////////////////////////////////////////////////////////////////////// - // Methods based on Vertex Name // - ////////////////////////////////////////////////////////////////////////////// - + + ////////////////////////////////////////////////////////////////////////////// + // Methods based on Vertex Name // + ////////////////////////////////////////////////////////////////////////////// + private static final String INITIAL_MAP_VERTEX_NAME = "initialmap"; private static final String FINAL_REDUCE_VERTEX_NAME = "finalreduce"; private static final String INTERMEDIATE_TASK_VERTEX_NAME_PREFIX = "ivertex"; @@ -70,7 +70,7 @@ public static String getIntermediateStageVertexName(int stageNum) { @Private static Configuration extractStageConf(Configuration baseConf, - String prefix) { + String prefix) { Configuration strippedConf = new Configuration(false); Configuration conf = new Configuration(false); Iterator> confEntries = baseConf.iterator(); diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java index 6f9c1c7600..1d01b4c106 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/TezTypeConverters.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -32,7 +32,7 @@ public class TezTypeConverters { // TODO Remove unused methods - + // Tez objects will be imported. Others will use the fully qualified name when // required. // All public methods named toYarn / toTez / toMapReduce @@ -40,9 +40,8 @@ public class TezTypeConverters { public static org.apache.hadoop.mapreduce.JobID toJobID(TezDAGID id) { return new JobID( String.valueOf(id.getApplicationId().getClusterTimestamp()), id.getId()); - } - + public static TaskAttemptId toYarn(TezTaskAttemptID taskAttemptId) { TaskAttemptID mrTaskAttemptId = IDConverter .toMRTaskAttemptId(taskAttemptId); @@ -68,7 +67,6 @@ public static Counters fromTez(TezCounters tezCounters) { Counter counter = counters.findCounter(xGrp.getName(), xCounter.getName()); counter.setValue(xCounter.getValue()); - } } return counters; diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/JobContextImpl.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/JobContextImpl.java index 2f0698699f..5a2d391d1e 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/JobContextImpl.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/JobContextImpl.java @@ -28,14 +28,14 @@ @InterfaceAudience.Private @InterfaceStability.Unstable -public class JobContextImpl - extends org.apache.hadoop.mapreduce.task.JobContextImpl +public class JobContextImpl + extends org.apache.hadoop.mapreduce.task.JobContextImpl implements JobContext { private JobConf job; private Progressable progress; public JobContextImpl(JobConf conf, TezDAGID dagId, - Progressable progress) { + Progressable progress) { super(conf, IDConverter.toMRJobId(dagId)); this.job = conf; this.progress = progress; @@ -44,20 +44,20 @@ public JobContextImpl(JobConf conf, TezDAGID dagId, public JobContextImpl(JobConf conf, TezDAGID dagId) { this(conf, dagId, Reporter.NULL); } - + /** * Get the job Configuration - * + * * @return JobConf */ public JobConf getJobConf() { return job; } - + /** * Get the progress mechanism for reporting progress. - * - * @return progress mechanism + * + * @return progress mechanism */ public Progressable getProgressible() { return progress; diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/MRCounters.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/MRCounters.java index b6b8e150ea..340b6c49fd 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/MRCounters.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/MRCounters.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.mapreduce.hadoop.mapred; @@ -30,7 +30,7 @@ public class MRCounters extends org.apache.hadoop.mapred.Counters { private final org.apache.tez.common.counters.TezCounters raw; - + public MRCounters(org.apache.tez.common.counters.TezCounters raw) { this.raw = raw; } @@ -43,14 +43,15 @@ public synchronized org.apache.hadoop.mapred.Counters.Group getGroup(String grou @SuppressWarnings("unchecked") @Override public synchronized Collection getGroupNames() { - return IteratorUtils.toList(raw.getGroupNames().iterator()); } + return IteratorUtils.toList(raw.getGroupNames().iterator()); + } @Override public synchronized String makeCompactString() { StringBuilder builder = new StringBuilder(); boolean first = true; - for(Group group: this){ - for(Counter counter: group) { + for (Group group : this) { + for (Counter counter : group) { if (first) { first = false; } else { @@ -95,7 +96,7 @@ public synchronized long getCounter(Enum key) { @Override public synchronized void incrAllCounters( org.apache.hadoop.mapred.Counters other) { - for (Group otherGroup: other) { + for (Group otherGroup : other) { Group group = getGroup(otherGroup.getName()); group.setDisplayName(otherGroup.getDisplayName()); for (Counter otherCounter : otherGroup) { @@ -116,59 +117,72 @@ public int size() { public String makeEscapedCompactString() { return toEscapedCompactString(this); } - + public static class MRCounterGroup extends org.apache.hadoop.mapred.Counters.Group { private final org.apache.tez.common.counters.CounterGroup group; + public MRCounterGroup(org.apache.tez.common.counters.CounterGroup group) { this.group = group; } + @Override public String getName() { return group.getName(); } + @Override public String getDisplayName() { return group.getDisplayName(); } + @Override public void setDisplayName(String displayName) { group.setDisplayName(displayName); } + @Override public void addCounter(org.apache.hadoop.mapred.Counters.Counter counter) { group.addCounter(convert(counter)); } + @Override public org.apache.hadoop.mapred.Counters.Counter addCounter(String name, - String displayName, long value) { + String displayName, long value) { return new MRCounter(group.addCounter(name, displayName, value)); } + @Override public org.apache.hadoop.mapred.Counters.Counter findCounter( String counterName, String displayName) { return new MRCounter(group.findCounter(counterName, displayName)); } + @Override public int size() { return group.size(); } + @SuppressWarnings("unchecked") @Override public org.apache.hadoop.mapreduce.counters.CounterGroupBase getUnderlyingGroup() { return new MRCounterGroup(group).getUnderlyingGroup(); } + @Override public void incrAllCounters( org.apache.hadoop.mapreduce.counters.CounterGroupBase rightGroup) { new MRCounterGroup(group).incrAllCounters(rightGroup); } + @Override public void readFields(DataInput arg0) throws IOException { } + @Override public void write(DataOutput arg0) throws IOException { } + @SuppressWarnings("unchecked") @Override public Iterator iterator() { @@ -204,10 +218,10 @@ public int hashCode() { return result; } } - + public static class MRCounter extends Counter { private final org.apache.tez.common.counters.TezCounter raw; - + public MRCounter(org.apache.tez.common.counters.TezCounter raw) { this.raw = raw; } @@ -265,7 +279,6 @@ public boolean contentEquals(Counter counter) { return c.equals(counter.getUnderlyingCounter()); } - @Override public long getCounter() { return raw.getValue(); @@ -294,24 +307,24 @@ static org.apache.tez.common.counters.TezCounter convert( org.apache.hadoop.mapreduce.Counter underlyingCounter = counter.getUnderlyingCounter(); if (underlyingCounter instanceof org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup.FrameworkCounter) { - org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup.FrameworkCounter - real = - (org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup.FrameworkCounter)underlyingCounter; + org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup.FrameworkCounter + real = + (org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup.FrameworkCounter) underlyingCounter; return new org.apache.tez.common.counters.FrameworkCounterGroup.FrameworkCounter( real.getKey(), real.getGroupName()); } else if (underlyingCounter instanceof org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup.FSCounter) { - org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup.FSCounter real = - (org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup.FSCounter)underlyingCounter; + org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup.FSCounter real = + (org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup.FSCounter) underlyingCounter; return new org.apache.tez.common.counters.FileSystemCounterGroup.FSCounter( real.getScheme(), convert(real.getFileSystemCounter())); } else { return new org.apache.tez.common.counters.GenericCounter( - underlyingCounter.getName(), - underlyingCounter.getDisplayName(), + underlyingCounter.getName(), + underlyingCounter.getDisplayName(), underlyingCounter.getValue()); } } - + static org.apache.tez.common.counters.FileSystemCounter convert( org.apache.hadoop.mapreduce.FileSystemCounter c) { switch (c) { @@ -328,6 +341,5 @@ static org.apache.tez.common.counters.FileSystemCounter convert( default: throw new IllegalArgumentException("Unknow FileSystemCounter: " + c); } - } } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/MRReporter.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/MRReporter.java index 28c8369166..55a9a54f72 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/MRReporter.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/MRReporter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,8 +29,7 @@ public class MRReporter implements Reporter { private TezCounters tezCounters; private InputSplit split; private float progress = 0f; - - + public MRReporter(TezCounters tezCounters) { this(tezCounters, null); } @@ -39,7 +38,7 @@ public MRReporter(TezCounters tezCounters, InputSplit split) { this.tezCounters = tezCounters; this.split = split; } - + @Override public void progress() { //TODO NEWTEZ @@ -84,11 +83,10 @@ public InputSplit getInputSplit() throws UnsupportedOperationException { public void setProgress(float progress) { this.progress = progress; } - + @Override public float getProgress() { // TODO NEWTEZ This is likely broken. Only set on task complete in Map/ReduceProcessor return this.progress; } - } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/TaskAttemptContextImpl.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/TaskAttemptContextImpl.java index fc32825674..0a8d21952e 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/TaskAttemptContextImpl.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/TaskAttemptContextImpl.java @@ -29,34 +29,34 @@ @InterfaceAudience.Private @InterfaceStability.Unstable public class TaskAttemptContextImpl - extends org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl - implements TaskAttemptContext { + extends org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl + implements TaskAttemptContext { private MRTaskReporter reporter; // FIXME we need to use DAG Id but we are using App Id public TaskAttemptContextImpl(JobConf conf, TaskAttemptID taskAttemptId, - MRTaskReporter reporter) { + MRTaskReporter reporter) { super(conf, taskAttemptId); this.reporter = reporter; } - + /** * Get the taskAttemptID. - * + * * @return TaskAttemptID */ public TaskAttemptID getTaskAttemptID() { return (TaskAttemptID) super.getTaskAttemptID(); } - + public Progressable getProgressible() { return reporter; } - + public JobConf getJobConf() { return (JobConf) getConfiguration(); } - + @Override public float getProgress() { return reporter.getProgress(); @@ -88,6 +88,4 @@ public void setStatus(String status) { setStatusString(status); reporter.setStatus(status); } - - } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/package-info.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/package-info.java index 73874b842d..61774c05a8 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/package-info.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapred/package-info.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,4 +19,4 @@ @Private package org.apache.tez.mapreduce.hadoop.mapred; -import org.apache.hadoop.classification.InterfaceAudience.Private; \ No newline at end of file +import org.apache.hadoop.classification.InterfaceAudience.Private; diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java index 1a0277c5d8..419027a6cf 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/JobContextImpl.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -65,10 +65,10 @@ public class JobContextImpl implements JobContext { public JobContextImpl(Configuration conf, TezDAGID dagId) { this(conf, dagId, MRTaskReporter.NULL); } - + public JobContextImpl(Configuration conf, TezDAGID dagId, Progressable progress) { if (conf instanceof JobConf) { - this.conf = (JobConf)conf; + this.conf = (JobConf) conf; } else { this.conf = new JobConf(conf); } @@ -92,14 +92,14 @@ public Configuration getConfiguration() { public JobID getJobID() { return IDConverter.toMRJobId(dagId); } - + /** * Set the JobID. */ public void setJobID(JobID jobId) { this.dagId = IDConverter.fromMRJobId(jobId); } - + /** * Get configured the number of reduce tasks for this job. Defaults to * 1. @@ -108,10 +108,10 @@ public void setJobID(JobID jobId) { public int getNumReduceTasks() { return conf.getNumReduceTasks(); } - + /** * Get the current working directory for the default file system. - * + * * @return the directory name. */ public Path getWorkingDirectory() throws IOException { @@ -125,7 +125,7 @@ public Path getWorkingDirectory() throws IOException { public Class getOutputKeyClass() { return conf.getOutputKeyClass(); } - + /** * Get the value class for job outputs. * @return the value class for job outputs. @@ -148,7 +148,7 @@ public Class getMapOutputKeyClass() { * Get the value class for the map output data. If it is not set, use the * (final) output value class This allows the map output value class to be * different than the final output value class. - * + * * @return the map output value class. */ public Class getMapOutputValueClass() { @@ -158,7 +158,7 @@ public Class getMapOutputValueClass() { /** * Get the user-specified job name. This is only used to identify the * job to the user. - * + * * @return the job's name, defaulting to "". */ public String getJobName() { @@ -167,79 +167,79 @@ public String getJobName() { /** * Get the {@link InputFormat} class for the job. - * + * * @return the {@link InputFormat} class for the job. */ @SuppressWarnings("unchecked") - public Class> getInputFormatClass() - throws ClassNotFoundException { - return (Class>) - conf.getClass(INPUT_FORMAT_CLASS_ATTR, TextInputFormat.class); + public Class> getInputFormatClass() + throws ClassNotFoundException { + return (Class>) + conf.getClass(INPUT_FORMAT_CLASS_ATTR, TextInputFormat.class); } /** * Get the {@link Mapper} class for the job. - * + * * @return the {@link Mapper} class for the job. */ @SuppressWarnings("unchecked") - public Class> getMapperClass() - throws ClassNotFoundException { - return (Class>) - conf.getClass(MAP_CLASS_ATTR, Mapper.class); + public Class> getMapperClass() + throws ClassNotFoundException { + return (Class>) + conf.getClass(MAP_CLASS_ATTR, Mapper.class); } /** * Get the combiner class for the job. - * + * * @return the combiner class for the job. */ @SuppressWarnings("unchecked") - public Class> getCombinerClass() - throws ClassNotFoundException { - return (Class>) - conf.getClass(COMBINE_CLASS_ATTR, null); + public Class> getCombinerClass() + throws ClassNotFoundException { + return (Class>) + conf.getClass(COMBINE_CLASS_ATTR, null); } /** * Get the {@link Reducer} class for the job. - * + * * @return the {@link Reducer} class for the job. */ @SuppressWarnings("unchecked") - public Class> getReducerClass() - throws ClassNotFoundException { - return (Class>) - conf.getClass(REDUCE_CLASS_ATTR, Reducer.class); + public Class> getReducerClass() + throws ClassNotFoundException { + return (Class>) + conf.getClass(REDUCE_CLASS_ATTR, Reducer.class); } /** * Get the {@link OutputFormat} class for the job. - * + * * @return the {@link OutputFormat} class for the job. */ @SuppressWarnings("unchecked") - public Class> getOutputFormatClass() - throws ClassNotFoundException { - return (Class>) - conf.getClass(OUTPUT_FORMAT_CLASS_ATTR, TextOutputFormat.class); + public Class> getOutputFormatClass() + throws ClassNotFoundException { + return (Class>) + conf.getClass(OUTPUT_FORMAT_CLASS_ATTR, TextOutputFormat.class); } /** * Get the {@link Partitioner} class for the job. - * + * * @return the {@link Partitioner} class for the job. */ @SuppressWarnings("unchecked") - public Class> getPartitionerClass() - throws ClassNotFoundException { - return (Class>) - conf.getClass(PARTITIONER_CLASS_ATTR, HashPartitioner.class); + public Class> getPartitionerClass() + throws ClassNotFoundException { + return (Class>) + conf.getClass(PARTITIONER_CLASS_ATTR, HashPartitioner.class); } /** * Get the {@link RawComparator} comparator used to compare keys. - * + * * @return the {@link RawComparator} comparator used to compare keys. */ public RawComparator getSortComparator() { @@ -265,30 +265,30 @@ public RawComparator getCombinerKeyGroupingComparator() { return getSortComparator(); } - /** + /** * Get the user defined {@link RawComparator} comparator for * grouping keys of inputs to the reduce. - * + * * @return comparator set by the user for grouping values. * @see Job#setGroupingComparatorClass(Class) for details. */ public RawComparator getGroupingComparator() { return conf.getOutputValueGroupingComparator(); } - + /** * Get whether job-setup and job-cleanup is needed for the job - * - * @return boolean + * + * @return boolean */ public boolean getJobSetupCleanupNeeded() { return conf.getBoolean(MRJobConfig.SETUP_CLEANUP_NEEDED, true); } - + /** * Get whether task-cleanup is needed for the job - * - * @return boolean + * + * @return boolean */ public boolean getTaskCleanupNeeded() { return conf.getBoolean(MRJobConfig.TASK_CLEANUP_NEEDED, true); @@ -302,7 +302,7 @@ public boolean getTaskCleanupNeeded() { public boolean getSymlink() { return DistributedCache.getSymlink(conf); } - + /** * Get the archive entries in classpath as an array of Path */ @@ -335,7 +335,7 @@ public URI[] getCacheFiles() throws IOException { * @throws IOException */ public Path[] getLocalCacheArchives() - throws IOException { + throws IOException { return DistributedCache.getLocalCacheArchives(conf); } @@ -345,7 +345,7 @@ public Path[] getLocalCacheArchives() * @throws IOException */ public Path[] getLocalCacheFiles() - throws IOException { + throws IOException { return DistributedCache.getLocalCacheFiles(conf); } @@ -358,7 +358,7 @@ public Path[] getFileClassPaths() { /** * Parse a list of longs into strings. - * + * * @param timestamps * the list of longs to parse * @return a list of string that were parsed. same length as timestamps. @@ -394,22 +394,22 @@ public String[] getFileTimestamps() { return toTimestampStrs(DistributedCache.getFileTimestamps(conf)); } - /** + /** * Get the configured number of maximum attempts that will be made to run a * map task, as specified by the mapred.map.max.attempts * property. If this property is not already set, the default is 4 attempts. - * + * * @return the max number of attempts per map task. */ public int getMaxMapAttempts() { return conf.getMaxMapAttempts(); } - /** + /** * Get the configured number of maximum attempts that will be made to run a * reduce task, as specified by the mapred.reduce.max.attempts * property. If this property is not already set, the default is 4 attempts. - * + * * @return the max number of attempts per reduce task. */ public int getMaxReduceAttempts() { @@ -429,7 +429,7 @@ public boolean getProfileEnabled() { * * The default value for this property is * "-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s" - * + * * @return the parameters to pass to the task child to configure profiling */ public String getProfileParams() { @@ -447,7 +447,7 @@ public IntegerRanges getProfileTaskRange(boolean isMap) { /** * Get the reported username for this job. - * + * * @return the username */ public String getUser() { @@ -467,5 +467,4 @@ public JobConf getJobConf() { public Progressable getProgressible() { return progress; } - } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/MapContextImpl.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/MapContextImpl.java index 87b0b31b3b..c897cd86fd 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/MapContextImpl.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/MapContextImpl.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -42,15 +42,15 @@ */ @InterfaceAudience.Private @InterfaceStability.Unstable -public class MapContextImpl - extends TaskInputOutputContextImpl +public class MapContextImpl + extends TaskInputOutputContextImpl implements MapContext { - private RecordReader reader; + private RecordReader reader; private InputSplit split; public MapContextImpl(Configuration conf, TaskAttemptID taskid, - RecordReader reader, - RecordWriter writer, + RecordReader reader, + RecordWriter writer, OutputCommitter committer, TaskContext context, InputSplit split, Reporter reporter) { @@ -80,6 +80,4 @@ public VALUEIN getCurrentValue() throws IOException, InterruptedException { public boolean nextKeyValue() throws IOException, InterruptedException { return reader.nextKeyValue(); } - } - diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskAttemptContextImpl.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskAttemptContextImpl.java index 941c8732bd..d0932fd09c 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskAttemptContextImpl.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskAttemptContextImpl.java @@ -37,11 +37,11 @@ @InterfaceAudience.Private @InterfaceStability.Unstable public class TaskAttemptContextImpl - extends org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl { + extends org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl { private final TezCounters tezCounters; private final Reporter reporter; - + public static org.apache.hadoop.mapred.TaskAttemptID createMockTaskAttemptID( long clusterId, int vertexIndex, int appId, int taskIndex, int taskAttemptNumber, boolean isMap) { return new org.apache.hadoop.mapred.TaskAttemptID( @@ -50,9 +50,9 @@ public static org.apache.hadoop.mapred.TaskAttemptID createMockTaskAttemptID( isMap ? TaskType.MAP : TaskType.REDUCE, taskIndex), taskAttemptNumber); } - - public static org.apache.hadoop.mapred.TaskAttemptID - createMockTaskAttemptIDFromTezTaskAttemptId(TezTaskAttemptID tezTaId, boolean isMap) { + + public static org.apache.hadoop.mapred.TaskAttemptID + createMockTaskAttemptIDFromTezTaskAttemptId(TezTaskAttemptID tezTaId, boolean isMap) { TezVertexID vId = tezTaId.getVertexID(); ApplicationId appId = vId.getApplicationId(); return new org.apache.hadoop.mapred.TaskAttemptID( @@ -61,32 +61,32 @@ public static org.apache.hadoop.mapred.TaskAttemptID createMockTaskAttemptID( isMap ? TaskType.MAP : TaskType.REDUCE, tezTaId.getTaskID().getId()), tezTaId.getId()); } - - public static org.apache.hadoop.mapred.TaskID - createMockTaskAttemptIDFromTezTaskId(TezTaskID tezTaId, boolean isMap) { + + public static org.apache.hadoop.mapred.TaskID + createMockTaskAttemptIDFromTezTaskId(TezTaskID tezTaId, boolean isMap) { TezVertexID vId = tezTaId.getVertexID(); ApplicationId appId = vId.getApplicationId(); return new org.apache.hadoop.mapred.TaskID(String.valueOf(appId.getClusterTimestamp()) - + String.valueOf(vId.getId()), appId.getId(), - isMap ? TaskType.MAP : TaskType.REDUCE, tezTaId.getId()); + + String.valueOf(vId.getId()), appId.getId(), + isMap ? TaskType.MAP : TaskType.REDUCE, tezTaId.getId()); } // FIXME we need to use DAG Id but we are using App Id public TaskAttemptContextImpl(Configuration conf, TezCounters tezCounters, long clusterId, - int vertexIndex, int appId, int taskIndex, int taskAttemptNumber, boolean isMap, - Reporter reporter) { + int vertexIndex, int appId, int taskIndex, int taskAttemptNumber, boolean isMap, + Reporter reporter) { // TODO NEWTEZ Can the jt Identifier string be taskContext.getUniqueId ? this(conf, createMockTaskAttemptID(clusterId, vertexIndex, appId, taskIndex, taskAttemptNumber, isMap), tezCounters, reporter); } //FIXME we need to use DAG Id but we are using App Id - public TaskAttemptContextImpl(Configuration conf, TaskAttemptID attemptId, - TezCounters tezCounters, boolean isMap, Reporter reporter) { - // TODO NEWTEZ Can the jt Identifier string be taskContext.getUniqueId ? - this(conf, attemptId, tezCounters, reporter); - } - + public TaskAttemptContextImpl(Configuration conf, TaskAttemptID attemptId, + TezCounters tezCounters, boolean isMap, Reporter reporter) { + // TODO NEWTEZ Can the jt Identifier string be taskContext.getUniqueId ? + this(conf, attemptId, tezCounters, reporter); + } + public TaskAttemptContextImpl(Configuration conf, TaskAttemptID taId, TezCounters tezCounters, Reporter reporter) { super(conf, taId); this.tezCounters = tezCounters; diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskInputOutputContextImpl.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskInputOutputContextImpl.java index cad73a809f..fa1880cecf 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskInputOutputContextImpl.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TaskInputOutputContextImpl.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -42,14 +42,14 @@ */ @InterfaceAudience.Private @InterfaceStability.Unstable -public abstract class TaskInputOutputContextImpl - extends TaskAttemptContextImpl - implements TaskInputOutputContext { - private RecordWriter output; +public abstract class TaskInputOutputContextImpl + extends TaskAttemptContextImpl + implements TaskInputOutputContext { + private RecordWriter output; private OutputCommitter committer; public TaskInputOutputContextImpl(Configuration conf, TaskAttemptID taskid, - RecordWriter output, + RecordWriter output, OutputCommitter committer, TaskContext context, Reporter reporter) { super(conf, taskid, context.getCounters(), reporter); @@ -61,17 +61,15 @@ public TaskInputOutputContextImpl(Configuration conf, TaskAttemptID taskid, * Advance to the next key, value pair, returning null if at end. * @return the key object that was read into, or null if no more */ - public abstract - boolean nextKeyValue() throws IOException, InterruptedException; - + public abstract boolean nextKeyValue() throws IOException, InterruptedException; + /** * Get the current key. * @return the current key object or null if there isn't one * @throws IOException * @throws InterruptedException */ - public abstract - KEYIN getCurrentKey() throws IOException, InterruptedException; + public abstract KEYIN getCurrentKey() throws IOException, InterruptedException; /** * Get the current value. @@ -79,14 +77,14 @@ public TaskInputOutputContextImpl(Configuration conf, TaskAttemptID taskid, * @throws IOException * @throws InterruptedException */ - public abstract VALUEIN getCurrentValue() throws IOException, - InterruptedException; + public abstract VALUEIN getCurrentValue() throws IOException, + InterruptedException; /** * Generate an output key/value pair. */ public void write(KEYOUT key, VALUEOUT value - ) throws IOException, InterruptedException { + ) throws IOException, InterruptedException { output.write(key, value); } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TezNullOutputCommitter.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TezNullOutputCommitter.java index 8de09c1be3..7be0e2a7c9 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TezNullOutputCommitter.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/TezNullOutputCommitter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/package-info.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/package-info.java index c12a8ac3fd..f174f2accb 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/package-info.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/mapreduce/package-info.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,4 +19,4 @@ @Private package org.apache.tez.mapreduce.hadoop.mapreduce; -import org.apache.hadoop.classification.InterfaceAudience.Private; \ No newline at end of file +import org.apache.hadoop.classification.InterfaceAudience.Private; diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/package-info.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/package-info.java index 3cc21db0f4..2a799223c9 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/package-info.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/package-info.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,4 +19,4 @@ @Private package org.apache.tez.mapreduce.hadoop; -import org.apache.hadoop.classification.InterfaceAudience.Private; \ No newline at end of file +import org.apache.hadoop.classification.InterfaceAudience.Private; diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java index e64d273b44..93164846db 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -80,29 +80,42 @@ * * It is compatible with all standard Apache Hadoop MapReduce * {@link InputFormat} implementations. - * + * * This class is not meant to be extended by external projects. */ @Public public class MRInput extends MRInputBase { - @Private public static final String TEZ_MAPREDUCE_DAG_INDEX = "tez.mapreduce.dag.index"; - @Private public static final String TEZ_MAPREDUCE_DAG_NAME = "tez.mapreduce.dag.name"; - @Private public static final String TEZ_MAPREDUCE_VERTEX_INDEX = "tez.mapreduce.vertex.index"; - @Private public static final String TEZ_MAPREDUCE_VERTEX_NAME = "tez.mapreduce.vertex.name"; - @Private public static final String TEZ_MAPREDUCE_TASK_INDEX = "tez.mapreduce.task.index"; - @Private public static final String TEZ_MAPREDUCE_TASK_ATTEMPT_INDEX = "tez.mapreduce.task.attempt.index"; - @Private public static final String TEZ_MAPREDUCE_INPUT_INDEX = "tez.mapreduce.input.index"; - @Private public static final String TEZ_MAPREDUCE_INPUT_NAME = "tez.mapreduce.input.name"; - @Private public static final String TEZ_MAPREDUCE_APPLICATION_ID = "tez.mapreduce.application.id"; - @Private public static final String TEZ_MAPREDUCE_UNIQUE_IDENTIFIER = "tez.mapreduce.unique.identifier"; - @Private public static final String TEZ_MAPREDUCE_DAG_ATTEMPT_NUMBER = "tez.mapreduce.dag.attempt.number"; - @Private public static final String TEZ_MAPREDUCE_DAG_ID= "tez.mapreduce.dag.id"; - @Private public static final String TEZ_MAPREDUCE_VERTEX_ID = "tez.mapreduce.vertex.id"; - @Private public static final String TEZ_MAPREDUCE_TASK_ID = "tez.mapreduce.task.id"; - @Private public static final String TEZ_MAPREDUCE_TASK_ATTEMPT_ID = "tez.mapreduce.task.attempt.id"; - - + @Private + public static final String TEZ_MAPREDUCE_DAG_INDEX = "tez.mapreduce.dag.index"; + @Private + public static final String TEZ_MAPREDUCE_DAG_NAME = "tez.mapreduce.dag.name"; + @Private + public static final String TEZ_MAPREDUCE_VERTEX_INDEX = "tez.mapreduce.vertex.index"; + @Private + public static final String TEZ_MAPREDUCE_VERTEX_NAME = "tez.mapreduce.vertex.name"; + @Private + public static final String TEZ_MAPREDUCE_TASK_INDEX = "tez.mapreduce.task.index"; + @Private + public static final String TEZ_MAPREDUCE_TASK_ATTEMPT_INDEX = "tez.mapreduce.task.attempt.index"; + @Private + public static final String TEZ_MAPREDUCE_INPUT_INDEX = "tez.mapreduce.input.index"; + @Private + public static final String TEZ_MAPREDUCE_INPUT_NAME = "tez.mapreduce.input.name"; + @Private + public static final String TEZ_MAPREDUCE_APPLICATION_ID = "tez.mapreduce.application.id"; + @Private + public static final String TEZ_MAPREDUCE_UNIQUE_IDENTIFIER = "tez.mapreduce.unique.identifier"; + @Private + public static final String TEZ_MAPREDUCE_DAG_ATTEMPT_NUMBER = "tez.mapreduce.dag.attempt.number"; + @Private + public static final String TEZ_MAPREDUCE_DAG_ID = "tez.mapreduce.dag.id"; + @Private + public static final String TEZ_MAPREDUCE_VERTEX_ID = "tez.mapreduce.vertex.id"; + @Private + public static final String TEZ_MAPREDUCE_TASK_ID = "tez.mapreduce.task.id"; + @Private + public static final String TEZ_MAPREDUCE_TASK_ATTEMPT_ID = "tez.mapreduce.task.attempt.id"; /** * Helper class to configure {@link MRInput} @@ -155,16 +168,16 @@ public static class MRInputConfigBuilder { initializeInputPath(); } } - + MRInputConfigBuilder setInputClassName(String className) { this.inputClassName = className; return this; } private MRInputConfigBuilder setInputPaths(String inputPaths) { - if (!(org.apache.hadoop.mapred.FileInputFormat.class.isAssignableFrom(inputFormat) || + if (!(org.apache.hadoop.mapred.FileInputFormat.class.isAssignableFrom(inputFormat) || FileInputFormat.class.isAssignableFrom(inputFormat))) { - throw new TezUncheckedException("When setting inputPaths the inputFormat must be " + + throw new TezUncheckedException("When setting inputPaths the inputFormat must be " + "assignable from either org.apache.hadoop.mapred.FileInputFormat or " + "org.apache.hadoop.mapreduce.lib.input.FileInputFormat. " + "Otherwise use the non-path configBuilder." + @@ -273,7 +286,7 @@ public DataSourceDescriptor build() { throw new TezUncheckedException(e); } } - + private DataSourceDescriptor createDistributorDataSource() throws IOException { InputSplitInfo inputSplitInfo; setupBasicConf(conf); @@ -331,7 +344,7 @@ private DataSourceDescriptor createCustomDataSource() throws IOException { private DataSourceDescriptor createGeneratorDataSource() throws IOException { setupBasicConf(conf); MRHelpers.translateMRConfToTez(conf); - + Collection uris = maybeGetURIsForCredentials(); UserPayload payload = MRInputHelpersInternal.createMRInputPayload( @@ -381,7 +394,6 @@ private Collection maybeGetURIsForCredentials() { } return null; } - } /** @@ -405,7 +417,7 @@ public static MRInputConfigBuilder createConfigBuilder(Configuration conf, } /** - * Create an {@link org.apache.tez.mapreduce.input.MRInput.MRInputConfigBuilder} + * Create an {@link org.apache.tez.mapreduce.input.MRInput.MRInputConfigBuilder} * for {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat} * or {@link org.apache.hadoop.mapred.FileInputFormat} format based InputFormats. *

    @@ -434,10 +446,10 @@ public static MRInputConfigBuilder createConfigBuilder(Configuration conf, } private static final Logger LOG = LoggerFactory.getLogger(MRInput.class); - + private final ReentrantLock rrLock = new ReentrantLock(); private final Condition rrInited = rrLock.newCondition(); - + private volatile boolean eventReceived = false; private boolean readerCreated = false; @@ -447,7 +459,7 @@ public static MRInputConfigBuilder createConfigBuilder(Configuration conf, protected TaskSplitIndex splitMetaInfo = new TaskSplitIndex(); // Potential counters - #splits, #totalSize, #actualyBytesRead - + @Private volatile boolean splitInfoViaEvents; @@ -479,15 +491,15 @@ void initializeInternal() throws IOException { // Primarily for visibility rrLock.lock(); try { - + if (splitInfoViaEvents) { if (useNewApi) { mrReader = new MRReaderMapReduce(jobConf, getContext().getCounters(), inputRecordCounter, getContext().getApplicationId().getClusterTimestamp(), getContext() - .getTaskVertexIndex(), getContext().getApplicationId().getId(), getContext() - .getTaskIndex(), getContext().getTaskAttemptNumber(), getContext()); + .getTaskVertexIndex(), getContext().getApplicationId().getId(), getContext() + .getTaskIndex(), getContext().getTaskAttemptNumber(), getContext()); } else { - mrReader = new MRReaderMapred(jobConf, getContext().getCounters(), inputRecordCounter, + mrReader = new MRReaderMapred(jobConf, getContext().getCounters(), inputRecordCounter, getContext()); } } else { @@ -608,7 +620,7 @@ public List close() throws IOException { * {@link MRInput} sets some additional parameters like split location when using * the new API. This methods returns the list of additional updates, and * should be used by Processors using the old MapReduce API with {@link MRInput}. - * + * * @return the additional fields set by {@link MRInput} */ public Configuration getConfigUpdates() { @@ -641,7 +653,7 @@ void processSplitEvent(InputDataInformationEvent event) rrLock.unlock(); } } - + void checkAndAwaitRecordReaderInitialization() throws IOException { assert rrLock.getHoldCount() == 1; rrLock.lock(); @@ -668,7 +680,7 @@ void initFromEvent(InputDataInformationEvent initEvent) rrLock.unlock(); } } - + private void initFromEventInternal(InputDataInformationEvent initEvent) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug(getContext().getInputOutputVertexNames() + " initializing RecordReader from event"); @@ -689,7 +701,6 @@ private void initFromEventInternal(InputDataInformationEvent initEvent) throws I LOG.debug(getContext().getInputOutputVertexNames() + " split Details -> SplitClass: " + split.getClass().getName() + ", NewSplit: " + split + ", length: " + splitLength); } - } else { org.apache.hadoop.mapred.InputSplit split = MRInputUtils.getOldSplitDetailsFromEvent(splitProto, jobConf); @@ -711,17 +722,16 @@ private void initFromEventInternal(InputDataInformationEvent initEvent) throws I private static class MRInputHelpersInternal extends MRInputHelpers { protected static UserPayload createMRInputPayload(Configuration conf, - boolean isGrouped, boolean isSorted) throws IOException { + boolean isGrouped, boolean isSorted) throws IOException { return MRInputHelpers.createMRInputPayload(conf, null, isGrouped, isSorted); } protected static UserPayload createMRInputPayload(Configuration conf, - MRRuntimeProtos.MRSplitsProto mrSplitsProto) throws + MRRuntimeProtos.MRSplitsProto mrSplitsProto) throws IOException { return MRInputHelpers.createMRInputPayload(conf, mrSplitsProto, false, true); } } - } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInputLegacy.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInputLegacy.java index bbf145eb6c..77d11308c2 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInputLegacy.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInputLegacy.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -38,13 +38,12 @@ public class MRInputLegacy extends MRInput { private static final Logger LOG = LoggerFactory.getLogger(MRInputLegacy.class); - + private InputDataInformationEvent initEvent; private volatile boolean inited = false; private ReentrantLock eventLock = new ReentrantLock(); private Condition eventCondition = eventLock.newCondition(); - /** * Create an {@link org.apache.tez.mapreduce.input.MRInput.MRInputConfigBuilder} * @param conf Configuration for the {@link MRInputLegacy} @@ -67,7 +66,7 @@ public static MRInputConfigBuilder createConfigBuilder(Configuration conf, Class return MRInput.createConfigBuilder(conf, inputFormat, inputPaths).setInputClassName( MRInputLegacy.class.getName()); } - + public MRInputLegacy(InputContext inputContext, int numPhysicalInputs) { super(inputContext, numPhysicalInputs); } @@ -76,11 +75,11 @@ public MRInputLegacy(InputContext inputContext, int numPhysicalInputs) { protected void initializeInternal() throws IOException { LOG.info(getContext().getInputOutputVertexNames() + " MRInputLegacy deferring initialization"); } - + @Private public org.apache.hadoop.mapreduce.InputSplit getNewInputSplit() { return (org.apache.hadoop.mapreduce.InputSplit) mrReader.getSplit(); - } + } @SuppressWarnings("rawtypes") @Unstable @@ -89,7 +88,7 @@ public org.apache.hadoop.mapreduce.RecordReader getNewRecordReader() { } public float getProgress() throws ProgressFailedException, InterruptedException { - return super.getProgress(); + return super.getProgress(); } @Private @@ -107,13 +106,13 @@ public boolean isUsingNewApi() { public RecordReader getOldRecordReader() { return (RecordReader) mrReader.getRecordReader(); } - + @LimitedPrivate("hive") public void init() throws IOException { super.initializeInternal(); checkAndAwaitRecordReaderInitialization(); } - + @Override void processSplitEvent(InputDataInformationEvent event) { eventLock.lock(); diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java index 4a98052211..98c233d5d7 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MultiMRInput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -94,13 +94,13 @@ public MultiMRInput(InputContext inputContext, int numPhysicalInputs) { * @return {@link MultiMRInputConfigBuilder} */ public static MultiMRInputConfigBuilder createConfigBuilder(Configuration conf, - @Nullable Class inputFormat) { + @Nullable Class inputFormat) { MultiMRInputConfigBuilder configBuilder = new MultiMRInputConfigBuilder(conf, inputFormat); configBuilder.setInputClassName(MultiMRInput.class.getName()).groupSplits(false); - + return configBuilder; } - + public static class MultiMRInputConfigBuilder extends MRInput.MRInputConfigBuilder { private MultiMRInputConfigBuilder(Configuration conf, Class inputFormat) { super(conf, inputFormat); diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/base/MRInputBase.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/base/MRInputBase.java index ccae0b1964..5071464a30 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/base/MRInputBase.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/base/MRInputBase.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -46,7 +46,6 @@ import java.io.IOException; import java.util.List; - @InterfaceAudience.Private public abstract class MRInputBase extends AbstractLogicalInput { @@ -87,7 +86,6 @@ public List initialize() throws IOException { } } - // Add tokens to the jobConf - in case they are accessed within the RR / IF jobConf.getCredentials().mergeAll(UserGroupInformation.getCurrentUser().getCredentials()); @@ -126,7 +124,6 @@ public List initialize() throws IOException { this.inputRecordCounter = getContext().getCounters().findCounter( TaskCounter.INPUT_RECORDS_PROCESSED); - return null; } } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRInputUtils.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRInputUtils.java index a2b87e04c9..7d5ae293b3 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRInputUtils.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRInputUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -59,7 +59,7 @@ public static org.apache.hadoop.mapreduce.InputSplit getNewSplitDetailsFromEvent return MRInputHelpers.createNewFormatSplitFromUserPayload( splitProto, serializationFactory); } - + @SuppressWarnings("unchecked") public static org.apache.hadoop.mapreduce.InputSplit getNewSplitDetailsFromDisk( TaskSplitIndex splitMetaInfo, JobConf jobConf, TezCounter splitBytesCounter) @@ -97,7 +97,7 @@ public static org.apache.hadoop.mapreduce.InputSplit getNewSplitDetailsFromDisk( @SuppressWarnings("unchecked") public static InputSplit getOldSplitDetailsFromDisk(TaskSplitIndex splitMetaInfo, - JobConf jobConf, TezCounter splitBytesCounter) throws IOException { + JobConf jobConf, TezCounter splitBytesCounter) throws IOException { Path file = new Path(splitMetaInfo.getSplitLocation()); FileSystem fs = FileSystem.getLocal(jobConf); file = fs.makeQualified(file); @@ -127,7 +127,7 @@ public static InputSplit getOldSplitDetailsFromDisk(TaskSplitIndex splitMetaInfo inFile.close(); return split; } - + @Private public static InputSplit getOldSplitDetailsFromEvent(MRSplitProto splitProto, Configuration conf) throws IOException { diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRReader.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRReader.java index aa35fec70c..a26eec0005 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRReader.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRReader.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,24 +26,29 @@ @Private public abstract class MRReader extends KeyValueReader { - + private final InputContext context; - + public abstract void setSplit(Object split) throws IOException; + public abstract boolean isSetup(); + public abstract float getProgress() throws IOException, InterruptedException; + public abstract void close() throws IOException; + public abstract Object getSplit(); + public abstract Object getRecordReader(); - + public MRReader(InputContext context) { this.context = context; } - + protected final void notifyProgress() { context.notifyProgress(); } - + protected final void notifyDone() { context.notifyProgress(); } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRReaderMapReduce.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRReaderMapReduce.java index ad3d4d67ab..d4946e4e10 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRReaderMapReduce.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRReaderMapReduce.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -51,15 +51,15 @@ public class MRReaderMapReduce extends MRReader { private boolean setupComplete = false; public MRReaderMapReduce(JobConf jobConf, TezCounters tezCounters, TezCounter inputRecordCounter, - long clusterId, int vertexIndex, int appId, int taskIndex, int taskAttemptNumber, InputContext context) + long clusterId, int vertexIndex, int appId, int taskIndex, int taskAttemptNumber, InputContext context) throws IOException { this(jobConf, null, tezCounters, inputRecordCounter, clusterId, vertexIndex, appId, taskIndex, taskAttemptNumber, context); } public MRReaderMapReduce(JobConf jobConf, InputSplit inputSplit, TezCounters tezCounters, - TezCounter inputRecordCounter, long clusterId, int vertexIndex, int appId, int taskIndex, - int taskAttemptNumber, InputContext context) throws IOException { + TezCounter inputRecordCounter, long clusterId, int vertexIndex, int appId, int taskIndex, + int taskAttemptNumber, InputContext context) throws IOException { super(context); this.inputRecordCounter = inputRecordCounter; this.taskAttemptContext = new TaskAttemptContextImpl(jobConf, tezCounters, clusterId, diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRReaderMapred.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRReaderMapred.java index e04ae7f046..9561237979 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRReaderMapred.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/MRReaderMapred.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -57,14 +57,14 @@ public class MRReaderMapred extends MRReader { private boolean setupComplete = false; - public MRReaderMapred(JobConf jobConf, TezCounters tezCounters, TezCounter inputRecordCounter, - InputContext context) + public MRReaderMapred(JobConf jobConf, TezCounters tezCounters, TezCounter inputRecordCounter, + InputContext context) throws IOException { this(jobConf, null, tezCounters, inputRecordCounter, context); } public MRReaderMapred(JobConf jobConf, InputSplit inputSplit, TezCounters tezCounters, - TezCounter inputRecordCounter, InputContext context) throws IOException { + TezCounter inputRecordCounter, InputContext context) throws IOException { super(context); this.jobConf = jobConf; this.tezCounters = tezCounters; @@ -142,7 +142,7 @@ public Object getCurrentValue() throws IOException { * {@link MRInput} sets some additional parameters like split location when using the new API. * This methods returns the list of additional updates, and should be used by Processors using the * old MapReduce API with {@link MRInput}. - * + * * @return the additional fields set by {@link MRInput} */ public Configuration getConfigUpdates() { diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/package-info.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/package-info.java index 04a9500c4c..7070dac765 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/package-info.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/lib/package-info.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,4 +19,4 @@ @Private package org.apache.tez.mapreduce.lib; -import org.apache.hadoop.classification.InterfaceAudience.Private; \ No newline at end of file +import org.apache.hadoop.classification.InterfaceAudience.Private; diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java index 9aeae25bd9..7951371fdc 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -99,7 +99,7 @@ public static class MROutputConfigBuilder { boolean doCommit = true; private MROutputConfigBuilder(Configuration conf, - Class outputFormatParam, boolean useLazyOutputFormat) { + Class outputFormatParam, boolean useLazyOutputFormat) { this.conf = conf; if (outputFormatParam != null) { outputFormatProvided = true; @@ -127,9 +127,9 @@ private MROutputConfigBuilder(Configuration conf, } else { throw new TezUncheckedException( "outputFormat must be assignable from either " + - "org.apache.hadoop.mapred.OutputFormat or " + - "org.apache.hadoop.mapreduce.OutputFormat" + - " Given: " + outputFormatParam.getName()); + "org.apache.hadoop.mapred.OutputFormat or " + + "org.apache.hadoop.mapreduce.OutputFormat" + + " Given: " + outputFormatParam.getName()); } } else { outputFormatProvided = false; @@ -147,7 +147,7 @@ private MROutputConfigBuilder(Configuration conf, this.outputFormat = conf.getClassByName(outputClass); Preconditions.checkState(org.apache.hadoop.mapreduce.OutputFormat.class .isAssignableFrom(this.outputFormat), "outputFormat must be assignable from " - + "org.apache.hadoop.mapreduce.OutputFormat"); + + "org.apache.hadoop.mapreduce.OutputFormat"); } else { String outputClass = conf.get("mapred.output.format.class"); if (StringUtils.isEmpty(outputClass)) { @@ -156,7 +156,7 @@ private MROutputConfigBuilder(Configuration conf, this.outputFormat = conf.getClassByName(outputClass); Preconditions.checkState(org.apache.hadoop.mapred.OutputFormat.class .isAssignableFrom(this.outputFormat), "outputFormat must be assignable from " - + "org.apache.hadoop.mapred.OutputFormat"); + + "org.apache.hadoop.mapred.OutputFormat"); } } catch (ClassNotFoundException e) { throw new TezUncheckedException(e); @@ -168,14 +168,14 @@ private MROutputConfigBuilder(Configuration conf, private MROutputConfigBuilder setOutputPath(String outputPath) { boolean passNewLazyOutputFormatCheck = (LazyOutputFormat.class.isAssignableFrom(outputFormat)) && - org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.class. - isAssignableFrom(conf.getClass( - MRJobConfig.LAZY_OUTPUTFORMAT_OUTPUTFORMAT, null)); + org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.class. + isAssignableFrom(conf.getClass( + MRJobConfig.LAZY_OUTPUTFORMAT_OUTPUTFORMAT, null)); boolean passOldLazyOutputFormatCheck = (org.apache.hadoop.mapred.lib.LazyOutputFormat.class. isAssignableFrom(outputFormat)) && - FileOutputFormat.class.isAssignableFrom(conf.getClass( - MRJobConfig.LAZY_OUTPUTFORMAT_OUTPUTFORMAT, null)); + FileOutputFormat.class.isAssignableFrom(conf.getClass( + MRJobConfig.LAZY_OUTPUTFORMAT_OUTPUTFORMAT, null)); if (!(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.class. isAssignableFrom(outputFormat) || @@ -200,7 +200,7 @@ private void initializeOutputPath() { outputPath = conf.get(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.OUTDIR); } } - + /** * Create the {@link DataSinkDescriptor} * @return {@link DataSinkDescriptor} @@ -243,7 +243,7 @@ public DataSinkDescriptor build() { } return ds; } - + /** * Get the credentials for the output from its {@link FileSystem}s * Use the method to turn this off when not using a {@link FileSystem} @@ -255,7 +255,7 @@ public MROutputConfigBuilder getCredentialsForSinkFileSystem(boolean value) { getCredentialsForSinkFilesystem = value; return this; } - + /** * Disable commit operations for the output (default: true) * If the value is set to false then no {@link org.apache.tez.runtime.api.OutputCommitter} will @@ -315,7 +315,7 @@ public static MROutputConfigBuilder createConfigBuilder(Configuration conf, } public static MROutputConfigBuilder createConfigBuilder(Configuration conf, - @Nullable Class outputFormat, boolean useLazyOutputFormat) { + @Nullable Class outputFormat, boolean useLazyOutputFormat) { return new MROutputConfigBuilder(conf, outputFormat, useLazyOutputFormat); } @@ -337,13 +337,13 @@ public static MROutputConfigBuilder createConfigBuilder(Configuration conf, * @return {@link org.apache.tez.mapreduce.output.MROutput.MROutputConfigBuilder} */ public static MROutputConfigBuilder createConfigBuilder(Configuration conf, - @Nullable Class outputFormat, @Nullable String outputPath) { + @Nullable Class outputFormat, @Nullable String outputPath) { return createConfigBuilder(conf, outputFormat, outputPath, false); } public static MROutputConfigBuilder createConfigBuilder(Configuration conf, - @Nullable Class outputFormat, @Nullable String outputPath, - boolean useLazyOutputFormat) { + @Nullable Class outputFormat, @Nullable String outputPath, + boolean useLazyOutputFormat) { MROutputConfigBuilder configurer = createConfigBuilder(conf, outputFormat, useLazyOutputFormat); if (outputPath != null) { configurer.setOutputPath(outputPath); @@ -355,7 +355,7 @@ public static MROutputConfigBuilder createConfigBuilder(Configuration conf, private final NumberFormat taskNumberFormat = NumberFormat.getInstance(); private final NumberFormat nonTaskNumberFormat = NumberFormat.getInstance(); - + protected JobConf jobConf; boolean useNewApi; protected AtomicBoolean flushed = new AtomicBoolean(false); @@ -421,9 +421,9 @@ protected List initializeBase() throws IOException, InterruptedException jobConf.set(JobContext.TASK_ID, taskAttemptId.getTaskID().toString()); jobConf.setBoolean(JobContext.TASK_ISMAP, isMapperOutput); jobConf.setInt(JobContext.TASK_PARTITION, - taskAttemptId.getTaskID().getId()); + taskAttemptId.getTaskID().getId()); jobConf.set(JobContext.ID, taskAttemptId.getJobID().toString()); - + String outputFormatClassName; outputRecordCounter = getContext().getCounters().findCounter( @@ -529,8 +529,8 @@ private TaskAttemptContext createTaskAttemptContext(TaskAttemptID attemptId) { protected String getOutputFileNamePrefix() { String prefix = jobConf.get(MRJobConfig.MROUTPUT_FILE_NAME_PREFIX); if (prefix == null) { - prefix = "part-v" + - nonTaskNumberFormat.format(getContext().getTaskVertexIndex()) + + prefix = "part-v" + + nonTaskNumberFormat.format(getContext().getTaskVertexIndex()) + "-o" + nonTaskNumberFormat.format(getContext().getOutputIndex()); } return prefix; @@ -557,7 +557,7 @@ public void write(Object key, Object value) throws IOException { newRecordWriter.write(key, value); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new IOInterruptedException("Interrupted while writing next key-value",e); + throw new IOInterruptedException("Interrupted while writing next key-value", e); } } else { oldRecordWriter.write(key, value); @@ -583,7 +583,7 @@ public synchronized List close() throws IOException { return null; } - + /** * Call this in the processor before finishing to ensure outputs that * outputs have been flushed. Must be called before commit. @@ -619,7 +619,6 @@ public void commit() throws IOException { } } - /** * MROutput expects that a Processor call abort in case of any error * ( including an error during commit ) prior to the Processor's completion @@ -633,5 +632,4 @@ public void abort() throws IOException { committer.abortTask(oldApiTaskAttemptContext); } } - } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutputLegacy.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutputLegacy.java index cac13d9813..0c16896de0 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutputLegacy.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutputLegacy.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MultiMROutput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MultiMROutput.java index 0bd573b6c3..380df317ae 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MultiMROutput.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MultiMROutput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,11 +18,11 @@ package org.apache.tez.mapreduce.output; - import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; + import org.apache.hadoop.mapreduce.OutputFormat; import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.util.ReflectionUtils; @@ -80,7 +80,7 @@ public List initialize() throws IOException, InterruptedException { * @return {@link org.apache.tez.mapreduce.output.MROutput.MROutputConfigBuilder} */ public static MROutputConfigBuilder createConfigBuilder(Configuration conf, - Class outputFormat, String outputPath, boolean useLazyOutputFormat) { + Class outputFormat, String outputPath, boolean useLazyOutputFormat) { return MROutput.createConfigBuilder(conf, outputFormat, outputPath, useLazyOutputFormat) .setOutputClassName(MultiMROutput.class.getName()); } @@ -104,7 +104,7 @@ public void write(Object key, Object value, String basePath) throw new UnsupportedOperationException( "Write without basePath isn't supported."); } - if (basePath.length() > 0 && basePath.charAt(0) == '/' ) { + if (basePath.length() > 0 && basePath.charAt(0) == '/') { // The base path can't be absolute path starting with "/". // Otherwise, it will cause the task temporary files being // written outside the output committer's task work path. @@ -118,7 +118,7 @@ public void write(Object key, Object value, String basePath) } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new IOInterruptedException( - "Interrupted while writing next key-value",e); + "Interrupted while writing next key-value", e); } } else { getOldRecordWriter(basePath).write(key, value); @@ -141,9 +141,9 @@ public void flush() throws IOException { } try { if (useNewApi) { - for (RecordWriter writer : newRecordWriters.values()) { - writer.close(newApiTaskAttemptContext); - } + for (RecordWriter writer : newRecordWriters.values()) { + writer.close(newApiTaskAttemptContext); + } } else { for (org.apache.hadoop.mapred.RecordWriter writer : oldRecordWriters.values()) { @@ -151,7 +151,7 @@ public void flush() throws IOException { } } } catch (InterruptedException e) { - throw new IOException("Interrupted while closing record writer", e); + throw new IOException("Interrupted while closing record writer", e); } } @@ -183,7 +183,7 @@ private synchronized RecordWriter getNewRecordWriter( @SuppressWarnings("unchecked") private synchronized org.apache.hadoop.mapred.RecordWriter - getOldRecordWriter(String baseFileName) throws IOException { + getOldRecordWriter(String baseFileName) throws IOException { // look for record-writer in the cache org.apache.hadoop.mapred.RecordWriter writer = @@ -191,10 +191,10 @@ private synchronized RecordWriter getNewRecordWriter( // If not in cache, create a new one if (writer == null) { - FileSystem fs = FileSystem.get(jobConf); - String finalName = getOutputName(baseFileName); - writer = oldOutputFormat.getRecordWriter(fs, jobConf, - finalName, new MRReporter(getContext().getCounters())); + FileSystem fs = FileSystem.get(jobConf); + String finalName = getOutputName(baseFileName); + writer = oldOutputFormat.getRecordWriter(fs, jobConf, + finalName, new MRReporter(getContext().getCounters())); // add the record-writer to the cache oldRecordWriters.put(baseFileName, writer); } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/partition/MRPartitioner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/partition/MRPartitioner.java index 80828d44e0..9718f36e2f 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/partition/MRPartitioner.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/partition/MRPartitioner.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -33,7 +33,7 @@ * Provides an implementation of {@link Partitioner} that is compatible * with Map Reduce partitioners. */ -@SuppressWarnings({ "rawtypes", "unchecked" }) +@SuppressWarnings({"rawtypes", "unchecked"}) @Public public class MRPartitioner implements org.apache.tez.runtime.library.api.Partitioner { diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java index 55e5709c3b..0341e1aa12 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTask.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.mapreduce.processor; @@ -99,7 +99,7 @@ public abstract class MRTask extends AbstractLogicalIOProcessor { protected TaskAttemptID taskAttemptId; protected Progress progress = new Progress(); protected SecretKey jobTokenSecret; - + LogicalInput input; LogicalOutput output; @@ -111,6 +111,7 @@ public abstract class MRTask extends AbstractLogicalIOProcessor { /** Construct output file names so that, when an output directory listing is * sorted lexicographically, positions correspond to output partitions.*/ private static final NumberFormat NUMBER_FORMAT = NumberFormat.getInstance(); + static { NUMBER_FORMAT.setMinimumIntegerDigits(5); NUMBER_FORMAT.setGroupingUsed(false); @@ -127,7 +128,7 @@ public MRTask(ProcessorContext processorContext, boolean isMap) { // TODO how to update progress @Override public void initialize() throws IOException, - InterruptedException { + InterruptedException { DeprecatedKeys.init(); @@ -144,14 +145,14 @@ public void initialize() throws IOException, UserPayload userPayload = processorContext.getUserPayload(); Configuration conf = TezUtils.createConfFromUserPayload(userPayload); if (conf instanceof JobConf) { - this.jobConf = (JobConf)conf; + this.jobConf = (JobConf) conf; } else { this.jobConf = new JobConf(conf); } jobConf.set(Constants.TEZ_RUNTIME_TASK_ATTEMPT_ID, taskAttemptId.toString()); jobConf.set(MRJobConfig.TASK_ATTEMPT_ID, - taskAttemptId.toString()); + taskAttemptId.toString()); jobConf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, processorContext.getDAGAttemptNumber()); @@ -246,8 +247,8 @@ private void configureLocalDirs() throws IOException { workDir = lDirAlloc.getLocalPathToRead("work", jobConf); } if (!madeDir) { - throw new IOException("Mkdirs failed to create " - + workDir.toString()); + throw new IOException("Mkdirs failed to create " + + workDir.toString()); } } // TODO NEWTEZ Is this required ? @@ -312,11 +313,11 @@ public ProcessorContext getUmbilical() { } public void initTask(LogicalOutput output) throws IOException, - InterruptedException { + InterruptedException { // By this time output has been initialized this.output = output; if (output instanceof MROutputLegacy) { - committer = ((MROutputLegacy)output).getOutputCommitter(); + committer = ((MROutputLegacy) output).getOutputCommitter(); } this.mrReporter = new MRTaskReporter(processorContext); this.useNewApi = jobConf.getUseNewMapper(); @@ -333,7 +334,7 @@ public MRTaskReporter getMRReporter() { return mrReporter; } - public TezCounters getCounters() { return counters; } + public TezCounters getCounters() {return counters;} public void setConf(JobConf jobConf) { this.jobConf = jobConf; @@ -359,7 +360,7 @@ public void done() throws IOException, InterruptedException { // TODO change this to use the new context // TODO TEZ Interaciton between Commit and OutputReady. Merge ? if (output instanceof MROutputLegacy) { - MROutputLegacy sOut = (MROutputLegacy)output; + MROutputLegacy sOut = (MROutputLegacy) output; if (sOut.isCommitRequired()) { //wait for commit approval and commit // TODO EVENTUALLY - Commit is not required for map tasks. @@ -398,7 +399,7 @@ private void commit(MROutputLegacy output) throws IOException { break; } Thread.sleep(1000); - } catch(InterruptedException ie) { + } catch (InterruptedException ie) { //ignore } catch (IOException ie) { LOG.warn("Failure sending canCommit: " @@ -426,13 +427,12 @@ private void commit(MROutputLegacy output) throws IOException { } } - private - void discardOutput(MROutputLegacy output) { + private void discardOutput(MROutputLegacy output) { try { output.abort(); - } catch (IOException ioe) { + } catch (IOException ioe) { LOG.warn("Failure cleaning up: " + - ExceptionUtils.getStackTrace(ioe)); + ExceptionUtils.getStackTrace(ioe)); } } @@ -450,16 +450,16 @@ public static String normalizeStatus(String status, Configuration conf) { return status; } - protected static - org.apache.hadoop.mapreduce.Reducer.Context + protected static + org.apache.hadoop.mapreduce.Reducer.Context createReduceContext(org.apache.hadoop.mapreduce.Reducer - reducer, + reducer, Configuration job, TaskAttemptID taskId, final TezRawKeyValueIterator rIter, org.apache.hadoop.mapreduce.Counter inputKeyCounter, org.apache.hadoop.mapreduce.Counter inputValueCounter, - org.apache.hadoop.mapreduce.RecordWriter output, + org.apache.hadoop.mapreduce.RecordWriter output, org.apache.hadoop.mapreduce.OutputCommitter committer, org.apache.hadoop.mapreduce.StatusReporter reporter, RawComparator comparator, @@ -494,25 +494,25 @@ public void close() throws IOException { } }; org.apache.hadoop.mapreduce.ReduceContext - reduceContext = - new ReduceContextImpl( - job, - taskId, - r, - inputKeyCounter, - inputValueCounter, - output, - committer, - reporter, - comparator, - keyClass, - valueClass); + reduceContext = + new ReduceContextImpl( + job, + taskId, + r, + inputKeyCounter, + inputValueCounter, + output, + committer, + reporter, + comparator, + keyClass, + valueClass); LOG.debug("Using key class: {}, valueClass: {}", keyClass, valueClass); - org.apache.hadoop.mapreduce.Reducer.Context + org.apache.hadoop.mapreduce.Reducer.Context reducerContext = - new WrappedReducer().getReducerContext( - reduceContext); + new WrappedReducer().getReducerContext( + reduceContext); return reducerContext; } @@ -535,14 +535,14 @@ public void localizeConfiguration(JobConf jobConf) jobConf.setInt(JobContext.TASK_PARTITION, taskAttemptId.getTaskID().getId()); jobConf.set(JobContext.ID, taskAttemptId.getJobID().toString()); - + jobConf.setBoolean(MRJobConfig.TASK_ISMAP, isMap); - + Path outputPath = FileOutputFormat.getOutputPath(jobConf); if (outputPath != null) { if ((committer instanceof FileOutputCommitter)) { - FileOutputFormat.setWorkOutputPath(jobConf, - ((FileOutputCommitter)committer).getTaskAttemptPath(taskAttemptContext)); + FileOutputFormat.setWorkOutputPath(jobConf, + ((FileOutputCommitter) committer).getTaskAttemptPath(taskAttemptContext)); } else { FileOutputFormat.setWorkOutputPath(jobConf, outputPath); } @@ -560,5 +560,4 @@ public JobContext getJobContext() { public TaskAttemptID getTaskAttemptId() { return taskAttemptId; } - } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTaskReporter.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTaskReporter.java index e3fdc27a93..b072056a65 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTaskReporter.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/MRTaskReporter.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.mapreduce.processor; @@ -55,9 +55,9 @@ public MRTaskReporter(OutputContext context) { this.reporter = new MRReporter(context.getCounters()); this.isProcessorContext = false; } - + public MRTaskReporter(InputContext context) { - this.context= context; + this.context = context; this.reporter = new MRReporter(context.getCounters()); this.isProcessorContext = false; } @@ -68,7 +68,7 @@ public void setProgress(float progress) { progress = ProgressHelper.processProgress(progress); reporter.setProgress(progress); if (isProcessorContext) { - ((ProcessorContext)context).setProgress(progress); + ((ProcessorContext) context).setProgress(progress); } else { // TODO FIXME NEWTEZ - will MROutput's reporter use this api? } @@ -80,7 +80,9 @@ public void setStatus(String status) { public float getProgress() { return reporter.getProgress(); - }; + } + + ; public void progress() { reporter.progress(); @@ -123,5 +125,4 @@ public InputSplit getInputSplit() throws UnsupportedOperationException { return split; } } - } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/SimpleMRProcessor.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/SimpleMRProcessor.java index d1ac107d9b..efca77e504 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/SimpleMRProcessor.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/SimpleMRProcessor.java @@ -34,8 +34,8 @@ /** * A {@link SimpleProcessor} that provides Map Reduce specific post - * processing by calling commit (if needed) on all {@link MROutput}s - * connected to this {@link Processor}. + * processing by calling commit (if needed) on all {@link MROutput}s + * connected to this {@link Processor}. */ @Public @Evolving diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java index 5cec62c466..b8b58aea73 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/map/MapProcessor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -54,9 +54,9 @@ import org.apache.tez.runtime.library.api.KeyValueWriter; import org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput; -@SuppressWarnings({ "unchecked", "rawtypes" }) +@SuppressWarnings({"unchecked", "rawtypes"}) @Private -public class MapProcessor extends MRTask{ +public class MapProcessor extends MRTask { private static final Logger LOG = LoggerFactory.getLogger(MapProcessor.class); @@ -82,17 +82,17 @@ public void close() throws IOException { @Override public void run(Map _inputs, - Map _outputs) throws Exception { + Map _outputs) throws Exception { this.inputs = _inputs; this.outputs = _outputs; progressHelper = new ProgressHelper(this.inputs, getContext(), this.getClass().getSimpleName()); LOG.info("Running map: " + processorContext.getUniqueIdentifier()); if (_inputs.size() != 1 - || _outputs.size() != 1) { + || _outputs.size() != 1) { throw new IOException("Cannot handle multiple _inputs or _outputs" - + ", inputCount=" + _inputs.size() - + ", outputCount=" + _outputs.size()); + + ", inputCount=" + _inputs.size() + + ", outputCount=" + _outputs.size()); } for (LogicalInput input : _inputs.values()) { @@ -112,7 +112,7 @@ public void run(Map _inputs, throw new IOException(new TezException( "Only MRInputLegacy supported. Input: " + in.getClass())); } - MRInputLegacy input = (MRInputLegacy)in; + MRInputLegacy input = (MRInputLegacy) in; input.init(); Configuration incrementalConf = input.getConfigUpdates(); if (incrementalConf != null) { @@ -123,9 +123,9 @@ public void run(Map _inputs, KeyValueWriter kvWriter = null; if ((out instanceof MROutputLegacy)) { - kvWriter = ((MROutputLegacy)out).getWriter(); - } else if ((out instanceof OrderedPartitionedKVOutput)){ - kvWriter = ((OrderedPartitionedKVOutput)out).getWriter(); + kvWriter = ((MROutputLegacy) out).getWriter(); + } else if ((out instanceof OrderedPartitionedKVOutput)) { + kvWriter = ((OrderedPartitionedKVOutput) out).getWriter(); } else { throw new IOException("Illegal output to map, outputClass=" + out.getClass()); @@ -139,7 +139,6 @@ public void run(Map _inputs, done(); } - /** * Update the job with details about the file split @@ -155,11 +154,11 @@ private void updateJobWithSplit(final JobConf job, InputSplit inputSplit) { } LOG.info("Processing mapred split: " + inputSplit); } - + private void updateJobWithSplit( final JobConf job, org.apache.hadoop.mapreduce.InputSplit inputSplit) { if (inputSplit instanceof org.apache.hadoop.mapreduce.lib.input.FileSplit) { - org.apache.hadoop.mapreduce.lib.input.FileSplit fileSplit = + org.apache.hadoop.mapreduce.lib.input.FileSplit fileSplit = (org.apache.hadoop.mapreduce.lib.input.FileSplit) inputSplit; job.set(JobContext.MAP_INPUT_FILE, fileSplit.getPath().toString()); job.setLong(JobContext.MAP_INPUT_START, fileSplit.getStart()); @@ -173,15 +172,15 @@ void runOldMapper( final MRTaskReporter reporter, final MRInputLegacy input, final KeyValueWriter output - ) throws IOException, InterruptedException { + ) throws IOException, InterruptedException { // Initialize input in-line since it sets parameters which may be used by the processor. // Done only for MRInput. // TODO use new method in MRInput to get required info //input.initialize(job, master); - + InputSplit inputSplit = input.getOldInputSplit(); - + updateJobWithSplit(job, inputSplit); RecordReader in = new OldRecordReader(input); @@ -189,10 +188,10 @@ void runOldMapper( OutputCollector collector = new OldOutputCollector(output); MapRunnable runner = - (MapRunnable)ReflectionUtils.newInstance(job.getMapRunnerClass(), job); + (MapRunnable) ReflectionUtils.newInstance(job.getMapRunnerClass(), job); + + runner.run(in, collector, (Reporter) reporter); - runner.run(in, collector, (Reporter)reporter); - // Set progress to 1.0f if there was no exception, reporter.setProgress(1.0f); // start the sort phase only if there are reducers @@ -200,10 +199,10 @@ void runOldMapper( } private void runNewMapper(final JobConf job, - MRTaskReporter reporter, - final MRInputLegacy in, - KeyValueWriter out - ) throws IOException, InterruptedException { + MRTaskReporter reporter, + final MRInputLegacy in, + KeyValueWriter out + ) throws IOException, InterruptedException { // Initialize input in-line since it sets parameters which may be used by the processor. // Done only for MRInput. @@ -230,16 +229,16 @@ private void runNewMapper(final JobConf job, new NewOutputCollector(out); org.apache.hadoop.mapreduce.InputSplit split = in.getNewInputSplit(); - + updateJobWithSplit(job, split); org.apache.hadoop.mapreduce.MapContext - mapContext = - new MapContextImpl( - job, taskAttemptId, - input, output, - committer, - processorContext, split, reporter); + mapContext = + new MapContextImpl( + job, taskAttemptId, + input, output, + committer, + processorContext, split, reporter); org.apache.hadoop.mapreduce.Mapper.Context mapperContext = new WrappedMapper().getMapContext(mapContext); @@ -248,7 +247,7 @@ private void runNewMapper(final JobConf job, mapper.run(mapperContext); // Set progress to 1.0f if there was no exception, reporter.setProgress(1.0f); - + this.statusUpdate(); input.close(); output.close(mapperContext); @@ -266,7 +265,7 @@ private NewRecordReader(MRInput in) throws IOException { @Override public void initialize(org.apache.hadoop.mapreduce.InputSplit split, - TaskAttemptContext context) throws IOException, + TaskAttemptContext context) throws IOException, InterruptedException { //in.initializeNewRecordReader(split, context); } @@ -295,10 +294,10 @@ public float getProgress() throws IOException, InterruptedException { return in.getProgress(); } catch (ProgressFailedException e) { if (e.getCause() instanceof IOException) { - throw (IOException)e.getCause(); + throw (IOException) e.getCause(); } if (e.getCause() instanceof InterruptedException) { - throw (InterruptedException)e.getCause(); + throw (InterruptedException) e.getCause(); } } throw new RuntimeException("Could not get Processor progress"); @@ -361,7 +360,7 @@ public float getProgress() throws IOException { } private static class OldOutputCollector - implements OutputCollector { + implements OutputCollector { private final KeyValueWriter output; OldOutputCollector(KeyValueWriter output) { @@ -369,12 +368,12 @@ private static class OldOutputCollector } public void collect(Object key, Object value) throws IOException { - output.write(key, value); + output.write(key, value); } } private static class NewOutputCollector - extends org.apache.hadoop.mapreduce.RecordWriter { + extends org.apache.hadoop.mapreduce.RecordWriter { private final KeyValueWriter out; NewOutputCollector(KeyValueWriter out) throws IOException { @@ -388,7 +387,7 @@ public void write(Object key, Object value) throws IOException, InterruptedExcep @Override public void close(TaskAttemptContext context - ) throws IOException, InterruptedException { + ) throws IOException, InterruptedException { } } diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java index 63b168f267..4a91c385ac 100644 --- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java +++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/processor/reduce/ReduceProcessor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -56,7 +56,7 @@ import org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput; @Private -@SuppressWarnings({ "unchecked", "rawtypes" }) +@SuppressWarnings({"unchecked", "rawtypes"}) public class ReduceProcessor extends MRTask { private static final Logger LOG = LoggerFactory.getLogger(ReduceProcessor.class); @@ -82,12 +82,11 @@ public void close() throws IOException { if (progressHelper != null) { progressHelper.shutDownProgressTaskService(); } - } @Override public void run(Map _inputs, - Map _outputs) throws Exception { + Map _outputs) throws Exception { this.inputs = _inputs; this.outputs = _outputs; progressHelper = new ProgressHelper(this.inputs, processorContext, this.getClass().getSimpleName()); @@ -135,11 +134,11 @@ public void run(Map _inputs, if (!(in instanceof OrderedGroupedInputLegacy)) { throw new IOException("Illegal input to reduce: " + in.getClass()); } - OrderedGroupedInputLegacy shuffleInput = (OrderedGroupedInputLegacy)in; + OrderedGroupedInputLegacy shuffleInput = (OrderedGroupedInputLegacy) in; KeyValuesReader kvReader = shuffleInput.getReader(); KeyValueWriter kvWriter = null; - if((out instanceof MROutputLegacy)) { + if ((out instanceof MROutputLegacy)) { kvWriter = ((MROutputLegacy) out).getWriter(); } else if ((out instanceof OrderedPartitionedKVOutput)) { kvWriter = ((OrderedPartitionedKVOutput) out).getWriter(); @@ -152,7 +151,7 @@ public void run(Map _inputs, runNewReducer( jobConf, mrReporter, - shuffleInput, comparator, keyClass, valueClass, + shuffleInput, comparator, keyClass, valueClass, kvWriter); } catch (ClassNotFoundException cnfe) { throw new IOException(cnfe); @@ -167,12 +166,12 @@ public void run(Map _inputs, } void runOldReducer(JobConf job, - final MRTaskReporter reporter, - KeyValuesReader input, - RawComparator comparator, - Class keyClass, - Class valueClass, - final KeyValueWriter output) throws IOException, InterruptedException { + final MRTaskReporter reporter, + KeyValuesReader input, + RawComparator comparator, + Class keyClass, + Class valueClass, + final KeyValueWriter output) throws IOException, InterruptedException { Reducer reducer = ReflectionUtils.newInstance(job.getReducerClass(), job); @@ -181,11 +180,11 @@ void runOldReducer(JobConf job, OutputCollector collector = new OutputCollector() { - public void collect(Object key, Object value) - throws IOException { - output.write(key, value); - } - }; + public void collect(Object key, Object value) + throws IOException { + output.write(key, value); + } + }; // apply reduce function try { @@ -202,7 +201,7 @@ public void collect(Object key, Object value) // Set progress to 1.0f if there was no exception, reporter.setProgress(1.0f); - + //Clean up: repeated in catch block below reducer.close(); //End of clean up. @@ -216,18 +215,18 @@ public void collect(Object key, Object value) } } - private static class ReduceValuesIterator - implements Iterator { + private static class ReduceValuesIterator + implements Iterator { private Counter reduceInputValueCounter; private KeyValuesReader in; private Progressable reporter; private Object currentKey; private Iterator currentValues; - public ReduceValuesIterator (KeyValuesReader in, - Progressable reporter, - Counter reduceInputValueCounter) - throws IOException { + public ReduceValuesIterator(KeyValuesReader in, + Progressable reporter, + Counter reduceInputValueCounter) + throws IOException { this.reduceInputValueCounter = reduceInputValueCounter; this.in = in; this.reporter = reporter; @@ -235,7 +234,7 @@ public ReduceValuesIterator (KeyValuesReader in, public boolean more() throws IOException { boolean more = in.next(); - if(more) { + if (more) { currentKey = in.getCurrentKey(); currentValues = in.getCurrentValues().iterator(); } else { @@ -268,17 +267,16 @@ public VALUE next() { public void remove() { throw new UnsupportedOperationException(); } - } void runNewReducer(JobConf job, - final MRTaskReporter reporter, - OrderedGroupedInputLegacy input, - RawComparator comparator, - Class keyClass, - Class valueClass, - final KeyValueWriter out - ) throws IOException, InterruptedException, ClassNotFoundException, TezException { + final MRTaskReporter reporter, + OrderedGroupedInputLegacy input, + RawComparator comparator, + Class keyClass, + Class valueClass, + final KeyValueWriter out + ) throws IOException, InterruptedException, ClassNotFoundException, TezException { // make a task context so we can get the classes org.apache.hadoop.mapreduce.TaskAttemptContext taskContext = getTaskAttemptContext(); @@ -286,7 +284,7 @@ void runNewReducer(JobConf job, // make a reducer org.apache.hadoop.mapreduce.Reducer reducer = (org.apache.hadoop.mapreduce.Reducer) - ReflectionUtils.newInstance(taskContext.getReducerClass(), job); + ReflectionUtils.newInstance(taskContext.getReducerClass(), job); // wrap value iterator to report progress. final TezRawKeyValueIterator rawIter = input.getIterator(); @@ -294,9 +292,11 @@ void runNewReducer(JobConf job, public void close() throws IOException { rawIter.close(); } + public DataInputBuffer getKey() throws IOException { return rawIter.getKey(); } + public Progress getProgress() { return rawIter.getProgress(); } @@ -325,17 +325,17 @@ public boolean next() throws IOException { org.apache.hadoop.mapreduce.RecordWriter trackedRW = new org.apache.hadoop.mapreduce.RecordWriter() { - @Override - public void write(Object key, Object value) throws IOException, - InterruptedException { - out.write(key, value); - } + @Override + public void write(Object key, Object value) throws IOException, + InterruptedException { + out.write(key, value); + } - @Override - public void close(TaskAttemptContext context) throws IOException, - InterruptedException { - } - }; + @Override + public void close(TaskAttemptContext context) throws IOException, + InterruptedException { + } + }; org.apache.hadoop.mapreduce.Reducer.Context reducerContext = createReduceContext( @@ -347,8 +347,6 @@ public void close(TaskAttemptContext context) throws IOException, reporter, comparator, keyClass, valueClass); - - reducer.run(reducerContext); // Set progress to 1.0f if there was no exception, @@ -363,5 +361,4 @@ public void localizeConfiguration(JobConf jobConf) super.localizeConfiguration(jobConf); jobConf.setBoolean(JobContext.TASK_ISMAP, false); } - } diff --git a/tez-mapreduce/src/test/java/org/apache/hadoop/mapred/split/TestGroupedSplits.java b/tez-mapreduce/src/test/java/org/apache/hadoop/mapred/split/TestGroupedSplits.java index 9d700652a7..6de9104ae9 100644 --- a/tez-mapreduce/src/test/java/org/apache/hadoop/mapred/split/TestGroupedSplits.java +++ b/tez-mapreduce/src/test/java/org/apache/hadoop/mapred/split/TestGroupedSplits.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -67,7 +67,7 @@ public class TestGroupedSplits { private static final Logger LOG = - LoggerFactory.getLogger(TestGroupedSplits.class); + LoggerFactory.getLogger(TestGroupedSplits.class); private static JobConf defaultConf = new JobConf(); private static FileSystem localFs = null; @@ -83,19 +83,19 @@ public class TestGroupedSplits { @SuppressWarnings("deprecation") private static Path workDir = - new Path(new Path(System.getProperty("test.build.data", "/tmp")), - "TestCombineTextInputFormat").makeQualified(localFs); + new Path(new Path(System.getProperty("test.build.data", "/tmp")), + "TestCombineTextInputFormat").makeQualified(localFs); // A reporter that does nothing private static final Reporter voidReporter = Reporter.NULL; - @Test(timeout=10000) + @Test(timeout = 10000) public void testFormat() throws Exception { JobConf job = new JobConf(defaultConf); Random random = new Random(); long seed = random.nextLong(); - LOG.info("seed = "+seed); + LOG.info("seed = " + seed); random.setSeed(seed); localFs.delete(workDir, true); @@ -109,7 +109,7 @@ public void testFormat() throws Exception { // create a combined split for the files TextInputFormat wrappedFormat = new TextInputFormat(); wrappedFormat.configure(job); - TezGroupedSplitsInputFormat format = + TezGroupedSplitsInputFormat format = new TezGroupedSplitsInputFormat(); format.setConf(job); format.setDesiredNumberOfSplits(1); @@ -117,7 +117,7 @@ public void testFormat() throws Exception { LongWritable key = new LongWritable(); Text value = new Text(); for (int i = 0; i < 3; i++) { - int numSplits = random.nextInt(length/20)+1; + int numSplits = random.nextInt(length / 20) + 1; LOG.info("splitting: requesting = " + numSplits); InputSplit[] splits = format.getSplits(job, numSplits); LOG.info("splitting: got = " + splits.length); @@ -127,13 +127,13 @@ public void testFormat() throws Exception { assertEquals("We got more than one splits!", 1, splits.length); InputSplit split = splits[0]; assertEquals("It should be TezGroupedSplit", - TezGroupedSplit.class, split.getClass()); + TezGroupedSplit.class, split.getClass()); // check the split BitSet bits = new BitSet(length); LOG.debug("split= " + split); RecordReader reader = - format.getRecordReader(split, job, voidReporter); + format.getRecordReader(split, job, voidReporter); try { int count = 0; while (reader.next(key, value)) { @@ -141,13 +141,13 @@ public void testFormat() throws Exception { LOG.debug("read " + v); if (bits.get(v)) { LOG.warn("conflict with " + v + - " at position "+reader.getPos()); + " at position " + reader.getPos()); } assertFalse("Key in multiple partitions.", bits.get(v)); bits.set(v); count++; } - LOG.info("splits="+split+" count=" + count); + LOG.info("splits=" + split + " count=" + count); } finally { reader.close(); } @@ -174,17 +174,17 @@ private static Range[] createRanges(int length, int numFiles, Random random) { // generate a number of files with various lengths Range[] ranges = new Range[numFiles]; for (int i = 0; i < numFiles; i++) { - int start = i == 0 ? 0 : ranges[i-1].end; + int start = i == 0 ? 0 : ranges[i - 1].end; int end = i == numFiles - 1 ? - length : - (length/numFiles)*(2*i + 1)/2 + random.nextInt(length/numFiles) + 1; + length : + (length / numFiles) * (2 * i + 1) / 2 + random.nextInt(length / numFiles) + 1; ranges[i] = new Range(start, end); } return ranges; } private static void createFiles(int length, int numFiles, Random random) - throws IOException { + throws IOException { Range[] ranges = createRanges(length, numFiles, random); for (int i = 0; i < numFiles; i++) { @@ -215,12 +215,12 @@ private static void writeFile(FileSystem fs, Path name, stm.close(); } - private static List readSplit(InputFormat format, + private static List readSplit(InputFormat format, InputSplit split, JobConf job) throws IOException { List result = new ArrayList(); RecordReader reader = - format.getRecordReader(split, job, voidReporter); + format.getRecordReader(split, job, voidReporter); LongWritable key = reader.createKey(); Text value = reader.createValue(); while (reader.next(key, value)) { @@ -239,122 +239,121 @@ public static void beforeClass() { /** * Test using the gzip codec for reading */ - @Test(timeout=10000) + @Test(timeout = 10000) public void testGzip() throws IOException { JobConf job = new JobConf(defaultConf); CompressionCodec gzip = new GzipCodec(); ReflectionUtils.setConf(gzip, job); localFs.delete(workDir, true); writeFile(localFs, new Path(workDir, "part1.txt.gz"), gzip, - "the quick\nbrown\nfox jumped\nover\n the lazy\n dog\n"); + "the quick\nbrown\nfox jumped\nover\n the lazy\n dog\n"); writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip, - "is\ngzip\n"); + "is\ngzip\n"); writeFile(localFs, new Path(workDir, "part3.txt.gz"), gzip, "one\nmore\nsplit\n"); FileInputFormat.setInputPaths(job, workDir); TextInputFormat wrappedFormat = new TextInputFormat(); wrappedFormat.configure(job); - TezGroupedSplitsInputFormat format = + TezGroupedSplitsInputFormat format = new TezGroupedSplitsInputFormat(); format.setConf(job); format.setInputFormat(wrappedFormat); - + // TextInputFormat will produce 3 splits - for (int j=1; j<=3; ++j) { + for (int j = 1; j <= 3; ++j) { format.setDesiredNumberOfSplits(j); InputSplit[] splits = format.getSplits(job, 100); - if (j==1) { + if (j == 1) { // j==1 covers single split corner case // and does not do grouping assertEquals("compressed splits == " + j, j, splits.length); } List results = new ArrayList(); - for (int i=0; i read = readSplit(format, splits[i], job); results.addAll(read); } assertEquals("splits length", 11, results.size()); - + final String[] firstList = - {"the quick", "brown", "fox jumped", "over", " the lazy", " dog"}; + {"the quick", "brown", "fox jumped", "over", " the lazy", " dog"}; final String[] secondList = {"is", "gzip"}; final String[] thirdList = {"one", "more", "split"}; String first = results.get(0).toString(); int start = 0; switch (first.charAt(0)) { - case 't': - start = testResults(results, firstList, start); - break; - case 'i': - start = testResults(results, secondList, start); - break; - case 'o': - start = testResults(results, thirdList, start); - break; - default: - Assert.fail("unexpected first token - " + first); + case 't': + start = testResults(results, firstList, start); + break; + case 'i': + start = testResults(results, secondList, start); + break; + case 'o': + start = testResults(results, thirdList, start); + break; + default: + Assert.fail("unexpected first token - " + first); } } } private static int testResults(List results, String[] first, int start) { for (int i = 0; i < first.length; i++) { - assertEquals("splits["+i+"]", first[i], results.get(start+i).toString()); + assertEquals("splits[" + i + "]", first[i], results.get(start + i).toString()); } - return first.length+start; - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - @Test(timeout=10000) + return first.length + start; + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + @Test(timeout = 10000) public void testGroupedSplitSize() throws IOException { JobConf job = new JobConf(defaultConf); InputFormat mockWrappedFormat = mock(InputFormat.class); - TezGroupedSplitsInputFormat format = + TezGroupedSplitsInputFormat format = new TezGroupedSplitsInputFormat(); format.setConf(job); format.setInputFormat(mockWrappedFormat); - + job = (JobConf) TezSplitGrouper.newConfigBuilder(job) - .setGroupingSplitSize(50*1000*1000l, 500*1000*1000l) + .setGroupingSplitSize(50 * 1000 * 1000l, 500 * 1000 * 1000l) .build(); InputSplit mockSplit1 = mock(InputSplit.class); - when(mockSplit1.getLength()).thenReturn(10*1000*1000l); + when(mockSplit1.getLength()).thenReturn(10 * 1000 * 1000l); when(mockSplit1.getLocations()).thenReturn(null); int numSplits = 100; InputSplit[] mockSplits = new InputSplit[numSplits]; - for (int i=0; i innerSplits = split.getGroupedSplits(); int pos = -1; // splits in group maintain original order @@ -416,30 +415,30 @@ public void testMaintainSplitOrdering() throws IOException { pos = splitPos; } // last one is rack split - if (i==3) { + if (i == 3) { assertTrue(split.getRack() != null); } } } - @Test (timeout=5000) + @Test(timeout = 5000) public void testRepeatableSplits() throws IOException { int numLocations = 3; String[] locations = new String[numLocations]; - InputSplit[] origSplits = new InputSplit[numLocations*4]; + InputSplit[] origSplits = new InputSplit[numLocations * 4]; long splitLength = 100; - for (int i=0; i testSplits1 = gSplit1.getGroupedSplits(); TezGroupedSplit gSplit2 = ((TezGroupedSplit) groupedSplits2[i]); List testSplits2 = gSplit2.getGroupedSplits(); assertEquals(testSplits1.size(), testSplits2.size()); - for (int j=0; j format = + TezGroupedSplitsInputFormat format = new TezGroupedSplitsInputFormat(); format.setConf(job); format.setInputFormat(mockWrappedFormat); - + // put multiple splits with multiple copies in the same location String[] locations = {"common", "common", "common"}; int numSplits = 3; InputSplit[] mockSplits = new InputSplit[numSplits]; - for (int i=0; i splitSet = Sets.newHashSet(split.wrappedSplits); assertEquals(numSplits, splitSet.size()); } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - @Test(timeout=10000) + + @SuppressWarnings({"rawtypes", "unchecked"}) + @Test(timeout = 10000) public void testGroupedSplitWithBadLocations() throws IOException { JobConf job = new JobConf(defaultConf); InputFormat mockWrappedFormat = mock(InputFormat.class); - TezGroupedSplitsInputFormat format = + TezGroupedSplitsInputFormat format = new TezGroupedSplitsInputFormat(); format.setConf(job); format.setInputFormat(mockWrappedFormat); - + // put multiple splits with multiple copies in the same location int numSplits = 3; InputSplit[] mockSplits = new InputSplit[numSplits]; InputSplit mockSplit1 = mock(InputSplit.class); - when(mockSplit1.getLength()).thenReturn(10*1000*1000l); + when(mockSplit1.getLength()).thenReturn(10 * 1000 * 1000l); when(mockSplit1.getLocations()).thenReturn(null); mockSplits[0] = mockSplit1; InputSplit mockSplit2 = mock(InputSplit.class); - when(mockSplit2.getLength()).thenReturn(10*1000*1000l); - when(mockSplit2.getLocations()).thenReturn(new String[] {null}); + when(mockSplit2.getLength()).thenReturn(10 * 1000 * 1000l); + when(mockSplit2.getLocations()).thenReturn(new String[]{null}); mockSplits[1] = mockSplit2; InputSplit mockSplit3 = mock(InputSplit.class); - when(mockSplit3.getLength()).thenReturn(10*1000*1000l); - when(mockSplit3.getLocations()).thenReturn(new String[] {null, null}); + when(mockSplit3.getLength()).thenReturn(10 * 1000 * 1000l); + when(mockSplit3.getLocations()).thenReturn(new String[]{null, null}); mockSplits[2] = mockSplit3; when(mockWrappedFormat.getSplits(any(), anyInt())).thenReturn(mockSplits); - + format.setDesiredNumberOfSplits(1); InputSplit[] splits = format.getSplits(job, 1); assertEquals(1, splits.length); @@ -606,13 +603,13 @@ public void testGroupedSplitWithBadLocations() throws IOException { split.write(new DataOutputStream(bOut)); } - @SuppressWarnings({ "rawtypes", "unchecked" }) + @SuppressWarnings({"rawtypes", "unchecked"}) // No grouping - @Test(timeout=10000) + @Test(timeout = 10000) public void testGroupedSplitWithBadLocations2() throws IOException { JobConf job = new JobConf(defaultConf); InputFormat mockWrappedFormat = mock(InputFormat.class); - TezGroupedSplitsInputFormat format = + TezGroupedSplitsInputFormat format = new TezGroupedSplitsInputFormat(); format.setConf(job); format.setInputFormat(mockWrappedFormat); @@ -623,24 +620,24 @@ public void testGroupedSplitWithBadLocations2() throws IOException { int numSplits = 5; InputSplit[] mockSplits = new InputSplit[numSplits]; InputSplit mockSplit1 = mock(InputSplit.class); - when(mockSplit1.getLength()).thenReturn(100*1000*1000l); + when(mockSplit1.getLength()).thenReturn(100 * 1000 * 1000l); when(mockSplit1.getLocations()).thenReturn(null); mockSplits[0] = mockSplit1; InputSplit mockSplit2 = mock(InputSplit.class); - when(mockSplit2.getLength()).thenReturn(100*1000*1000l); - when(mockSplit2.getLocations()).thenReturn(new String[] {null}); + when(mockSplit2.getLength()).thenReturn(100 * 1000 * 1000l); + when(mockSplit2.getLocations()).thenReturn(new String[]{null}); mockSplits[1] = mockSplit2; InputSplit mockSplit3 = mock(InputSplit.class); - when(mockSplit3.getLength()).thenReturn(100*1000*1000l); - when(mockSplit3.getLocations()).thenReturn(new String[] {null, null}); + when(mockSplit3.getLength()).thenReturn(100 * 1000 * 1000l); + when(mockSplit3.getLocations()).thenReturn(new String[]{null, null}); mockSplits[2] = mockSplit3; InputSplit mockSplit4 = mock(InputSplit.class); - when(mockSplit4.getLength()).thenReturn(100*1000*1000l); - when(mockSplit4.getLocations()).thenReturn(new String[] {validLocation}); + when(mockSplit4.getLength()).thenReturn(100 * 1000 * 1000l); + when(mockSplit4.getLocations()).thenReturn(new String[]{validLocation}); mockSplits[3] = mockSplit4; InputSplit mockSplit5 = mock(InputSplit.class); - when(mockSplit5.getLength()).thenReturn(100*1000*1000l); - when(mockSplit5.getLocations()).thenReturn(new String[] {validLocation, null, validLocation2}); + when(mockSplit5.getLength()).thenReturn(100 * 1000 * 1000l); + when(mockSplit5.getLocations()).thenReturn(new String[]{validLocation, null, validLocation2}); mockSplits[4] = mockSplit4; when(mockWrappedFormat.getSplits(any(), anyInt())).thenReturn(mockSplits); @@ -648,14 +645,14 @@ public void testGroupedSplitWithBadLocations2() throws IOException { format.setDesiredNumberOfSplits(numSplits); InputSplit[] splits = format.getSplits(job, 1); assertEquals(numSplits, splits.length); - for (int i = 0 ; i < numSplits ; i++) { + for (int i = 0; i < numSplits; i++) { TezGroupedSplit split = (TezGroupedSplit) splits[i]; // all 3 splits are present assertEquals(1, split.wrappedSplits.size()); - if (i==3) { + if (i == 3) { assertEquals(1, split.getLocations().length); assertEquals(validLocation, split.getLocations()[0]); - } else if (i==4) { + } else if (i == 4) { assertEquals(1, split.getLocations().length); assertTrue(split.getLocations()[0].equals(validLocation) || split.getLocations()[0].equals(validLocation2)); } else { @@ -666,17 +663,17 @@ public void testGroupedSplitWithBadLocations2() throws IOException { } } - @SuppressWarnings({ "rawtypes", "unchecked" }) - @Test(timeout=10000) + @SuppressWarnings({"rawtypes", "unchecked"}) + @Test(timeout = 10000) public void testGroupedSplitWithEstimator() throws IOException { JobConf job = new JobConf(defaultConf); job = (JobConf) TezSplitGrouper.newConfigBuilder(job) - .setGroupingSplitSize(12*1000*1000l, 25*1000*1000l) + .setGroupingSplitSize(12 * 1000 * 1000l, 25 * 1000 * 1000l) .build(); InputFormat mockWrappedFormat = mock(InputFormat.class); - TezGroupedSplitsInputFormat format = + TezGroupedSplitsInputFormat format = new TezGroupedSplitsInputFormat(); format.setConf(job); format.setInputFormat(mockWrappedFormat); @@ -685,7 +682,7 @@ public void testGroupedSplitWithEstimator() throws IOException { final InputSplit mockSplit2 = mock(InputSplit.class); final InputSplit mockSplit3 = mock(InputSplit.class); - final String[] locations = new String[] { "common", "common", "common" }; + final String[] locations = new String[]{"common", "common", "common"}; final SplitSizeEstimator estimator = new SplitSizeEstimator() { @@ -709,8 +706,8 @@ public long getEstimatedSize(InputSplit split) throws IOException { // put multiple splits which should be grouped (1,1,2) Mb, but estimated to be 10x // 10,10,20Mb - grouped with min=12Mb, max=25Mb // should be grouped as (1,1),(2) - InputSplit[] mockSplits = new InputSplit[] { mockSplit1, mockSplit2, - mockSplit3 }; + InputSplit[] mockSplits = new InputSplit[]{mockSplit1, mockSplit2, + mockSplit3}; when(mockWrappedFormat.getSplits(any(), anyInt())) .thenReturn(mockSplits); @@ -734,30 +731,29 @@ public long getEstimatedSize(InputSplit split) throws IOException { } } - // Splits get grouped - @Test (timeout = 10000) + @Test(timeout = 10000) public void testGroupingWithCustomLocations1() throws IOException { int numSplits = 3; InputSplit[] mockSplits = new InputSplit[numSplits]; InputSplit mockSplit1 = mock(InputSplit.class); - when(mockSplit1.getLength()).thenReturn(100*1000*1000l); - when(mockSplit1.getLocations()).thenReturn(new String[] {"location1", "location2"}); + when(mockSplit1.getLength()).thenReturn(100 * 1000 * 1000l); + when(mockSplit1.getLocations()).thenReturn(new String[]{"location1", "location2"}); mockSplits[0] = mockSplit1; InputSplit mockSplit2 = mock(InputSplit.class); - when(mockSplit2.getLength()).thenReturn(100*1000*1000l); - when(mockSplit2.getLocations()).thenReturn(new String[] {"location3", "location4"}); + when(mockSplit2.getLength()).thenReturn(100 * 1000 * 1000l); + when(mockSplit2.getLocations()).thenReturn(new String[]{"location3", "location4"}); mockSplits[1] = mockSplit2; InputSplit mockSplit3 = mock(InputSplit.class); - when(mockSplit3.getLength()).thenReturn(100*1000*1000l); - when(mockSplit3.getLocations()).thenReturn(new String[] {"location5", "location6"}); + when(mockSplit3.getLength()).thenReturn(100 * 1000 * 1000l); + when(mockSplit3.getLocations()).thenReturn(new String[]{"location5", "location6"}); mockSplits[2] = mockSplit3; SplitLocationProvider locationProvider = new SplitLocationProvider() { @Override public String[] getLocations(InputSplit split) throws IOException { - return new String[] {"customLocation"}; + return new String[]{"customLocation"}; } }; @@ -767,8 +763,8 @@ public String[] getLocations(InputSplit split) throws IOException { // Sanity. 1 group, with 3 splits. assertEquals(1, groupedSplits.length); - assertTrue(groupedSplits[0] instanceof TezGroupedSplit); - TezGroupedSplit groupedSplit = (TezGroupedSplit)groupedSplits[0]; + assertTrue(groupedSplits[0] instanceof TezGroupedSplit); + TezGroupedSplit groupedSplit = (TezGroupedSplit) groupedSplits[0]; assertEquals(3, groupedSplit.getGroupedSplits().size()); // Verify that the split ends up being grouped to the custom location. @@ -777,28 +773,28 @@ public String[] getLocations(InputSplit split) throws IOException { } // Original splits returned. - @Test (timeout = 10000) + @Test(timeout = 10000) public void testGroupingWithCustomLocations2() throws IOException { int numSplits = 3; InputSplit[] mockSplits = new InputSplit[numSplits]; InputSplit mockSplit1 = mock(InputSplit.class); - when(mockSplit1.getLength()).thenReturn(100*1000*1000l); - when(mockSplit1.getLocations()).thenReturn(new String[] {"location1", "location2"}); + when(mockSplit1.getLength()).thenReturn(100 * 1000 * 1000l); + when(mockSplit1.getLocations()).thenReturn(new String[]{"location1", "location2"}); mockSplits[0] = mockSplit1; InputSplit mockSplit2 = mock(InputSplit.class); - when(mockSplit2.getLength()).thenReturn(100*1000*1000l); - when(mockSplit2.getLocations()).thenReturn(new String[] {"location3", "location4"}); + when(mockSplit2.getLength()).thenReturn(100 * 1000 * 1000l); + when(mockSplit2.getLocations()).thenReturn(new String[]{"location3", "location4"}); mockSplits[1] = mockSplit2; InputSplit mockSplit3 = mock(InputSplit.class); - when(mockSplit3.getLength()).thenReturn(100*1000*1000l); - when(mockSplit3.getLocations()).thenReturn(new String[] {"location5", "location6"}); + when(mockSplit3.getLength()).thenReturn(100 * 1000 * 1000l); + when(mockSplit3.getLocations()).thenReturn(new String[]{"location5", "location6"}); mockSplits[2] = mockSplit3; SplitLocationProvider locationProvider = new SplitLocationProvider() { @Override public String[] getLocations(InputSplit split) throws IOException { - return new String[] {"customLocation"}; + return new String[]{"customLocation"}; } }; @@ -808,9 +804,9 @@ public String[] getLocations(InputSplit split) throws IOException { // Sanity. 3 group, with 1 split each assertEquals(3, groupedSplits.length); - for (int i = 0 ; i < 3 ; i++) { - assertTrue(groupedSplits[i] instanceof TezGroupedSplit); - TezGroupedSplit groupedSplit = (TezGroupedSplit)groupedSplits[i]; + for (int i = 0; i < 3; i++) { + assertTrue(groupedSplits[i] instanceof TezGroupedSplit); + TezGroupedSplit groupedSplit = (TezGroupedSplit) groupedSplits[i]; assertEquals(1, groupedSplit.getGroupedSplits().size()); // Verify the splits have their final location set to customLocation @@ -917,5 +913,4 @@ private void verifySplitsFortestAllowSmallSplitsEarly(InputSplit[] groupedSplits assertEquals(2, entry.getValue().getValue()); } } - } diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilical.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilical.java index 9fe3ee0a8d..2ed553c84f 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilical.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TestUmbilical.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -75,5 +75,4 @@ public boolean canCommit(TezTaskAttemptID taskAttemptID) throws IOException { LOG.info("Got canCommit from task: " + taskAttemptID); return true; } - } diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java index 83c28dd7bb..5443ccef48 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/TezTestUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -41,18 +41,18 @@ public static TezTaskAttemptID getMockTaskAttemptId( TezVertexID.getInstance( TezDAGID.getInstance( ApplicationId.newInstance(0, jobId), jobId), - vertexId), - taskId) + vertexId), + taskId) , taskAttemptId); } - + public static TezTaskID getMockTaskId(int jobId, - int vertexId, int taskId) { + int vertexId, int taskId) { return TezTaskID.getInstance( TezVertexID.getInstance(TezDAGID.getInstance( ApplicationId.newInstance(0, jobId), jobId), vertexId), - taskId); + taskId); } public static class TezRootInputInitializerContextForTest implements @@ -137,7 +137,5 @@ public void addCounters(TezCounters tezCounters) { public UserPayload getUserPayload() { throw new UnsupportedOperationException("getUserPayload not implemented in this mock"); } - } - } diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/combine/TestMRCombiner.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/combine/TestMRCombiner.java index 19bb8a0941..1810ff026c 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/combine/TestMRCombiner.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/combine/TestMRCombiner.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -151,7 +151,7 @@ private static class TezRawKeyValueIteratorTest implements TezRawKeyValueIterator { private int i = -1; - private String[] keys = { "tez", "tez", "tez", "apache", "hadoop", "hadoop" }; + private String[] keys = {"tez", "tez", "tez", "apache", "hadoop", "hadoop"}; @Override public boolean next() throws IOException { @@ -164,7 +164,7 @@ public boolean next() throws IOException { } public boolean hasNext() throws IOException { - if (i < (keys.length - 1)) { + if (i < (keys.length - 1)) { return true; } return false; @@ -218,7 +218,7 @@ public void close() throws IOException { @Override public void reduce(Text key, Iterator value, - OutputCollector collector, Reporter reporter) + OutputCollector collector, Reporter reporter) throws IOException { int count = 0; while (value.hasNext()) { @@ -232,7 +232,7 @@ private static class NewReducer extends org.apache.hadoop.mapreduce.Reducer { @Override protected void reduce(Text key, Iterable values, - Context context) throws IOException, InterruptedException { + Context context) throws IOException, InterruptedException { int count = 0; for (IntWritable value : values) { count += value.get(); @@ -244,7 +244,7 @@ protected void reduce(Text key, Iterable values, private static class Top2OldReducer extends OldReducer { @Override public void reduce(Text key, Iterator value, - OutputCollector collector, Reporter reporter) + OutputCollector collector, Reporter reporter) throws IOException { int i = 0; while (value.hasNext()) { @@ -259,7 +259,7 @@ public void reduce(Text key, Iterator value, private static class Top2NewReducer extends NewReducer { @Override protected void reduce(Text key, Iterable values, - Context context) throws IOException, InterruptedException { + Context context) throws IOException, InterruptedException { int i = 0; for (IntWritable value : values) { if (i++ < 2) { diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputAMSplitGenerator.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputAMSplitGenerator.java index 9f6ac3b74f..b2641d0768 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputAMSplitGenerator.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputAMSplitGenerator.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -78,12 +78,12 @@ public void testGroupSplitsEnabledSortSplitsEnabled() @Test(timeout = 5000) public void testGroupSplitsEnabledSortSplitsDisabled() - throws Exception { + throws Exception { testGroupSplitsAndSortSplits(true, false); } private void testGroupSplitsAndSortSplits(boolean groupSplitsEnabled, - boolean sortSplitsEnabled) throws Exception { + boolean sortSplitsEnabled) throws Exception { Configuration conf = new Configuration(); String[] splitLengths = new String[50]; for (int i = 0; i < splitLengths.length; i++) { @@ -91,7 +91,7 @@ private void testGroupSplitsAndSortSplits(boolean groupSplitsEnabled, } conf.setStrings(SPLITS_LENGTHS, splitLengths); DataSourceDescriptor dataSource = MRInput.createConfigBuilder( - conf, InputFormatForTest.class). + conf, InputFormatForTest.class). groupSplits(groupSplitsEnabled).sortSplits(sortSplitsEnabled).build(); UserPayload userPayload = dataSource.getInputDescriptor().getUserPayload(); @@ -116,11 +116,11 @@ private void testGroupSplitsAndSortSplits(boolean groupSplitsEnabled, InputSplit is = MRInputUtils.getNewSplitDetailsFromEvent( eventProto, new Configuration()); if (groupSplitsEnabled) { - numRawInputSplits += ((TezGroupedSplit)is).getGroupedSplits().size(); - for (InputSplit inputSplit : ((TezGroupedSplit)is).getGroupedSplits()) { + numRawInputSplits += ((TezGroupedSplit) is).getGroupedSplits().size(); + for (InputSplit inputSplit : ((TezGroupedSplit) is).getGroupedSplits()) { assertTrue(inputSplit instanceof InputSplitForTest); } - assertTrue(((TezGroupedSplit)is).getGroupedSplits().get(0) + assertTrue(((TezGroupedSplit) is).getGroupedSplits().get(0) instanceof InputSplitForTest); } else { numRawInputSplits++; @@ -165,13 +165,13 @@ public void close() throws IOException { @Override public IntWritable getCurrentKey() throws IOException, - InterruptedException { + InterruptedException { return new IntWritable(0); } @Override public IntWritable getCurrentValue() throws IOException, - InterruptedException { + InterruptedException { return new IntWritable(0); } @@ -223,6 +223,7 @@ public InputSplitForTest() { public int getIdentifier() { return this.identifier; } + public InputSplitForTest(int identifier, int length) { this.identifier = identifier; this.length = length; @@ -247,7 +248,7 @@ public long getLength() throws IOException { @Override public String[] getLocations() throws IOException { - return new String[] {"localhost"}; + return new String[]{"localhost"}; } } } diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputSplitDistributor.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputSplitDistributor.java index 4aaa7e2e76..c93d00e20d 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputSplitDistributor.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputSplitDistributor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -180,6 +180,5 @@ public long getLength() throws IOException { public String[] getLocations() throws IOException { return null; } - } } diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestConfigTranslationMRToTez.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestConfigTranslationMRToTez.java index df68c8dff8..27caa98743 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestConfigTranslationMRToTez.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestConfigTranslationMRToTez.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -35,12 +35,12 @@ public class TestConfigTranslationMRToTez { public void testComplexKeys() { JobConf confVertex1 = new JobConf(); - + confVertex1.set(MRJobConfig.MAP_OUTPUT_KEY_CLASS, IntWritable.class.getName()); - + confVertex1.unset(MRJobConfig.KEY_COMPARATOR); confVertex1.unset(MRJobConfig.GROUP_COMPARATOR_CLASS); - + MRHelpers.translateMRConfToTez(confVertex1); assertEquals(IntWritable.Comparator.class.getName(), ConfigUtils diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java index c574e38b71..9fd46d88c5 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -163,5 +163,4 @@ public void verifyTezOverridenKeys() { assertNull(jobConf.get(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART)); assertNull(jobConf.get(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST)); } - } diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java index 9766cc060f..9bbacd8b69 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -116,7 +116,7 @@ private Configuration setupConfigForMREnvTest() { conf.set(MRJobConfig.MAP_LOG_LEVEL, "TRACE"); conf.set(MRJobConfig.REDUCE_LOG_LEVEL, "FATAL"); final String mapredAdminUserEnv = Shell.WINDOWS ? - "PATH=%PATH%" + File.pathSeparator + "%TEZ_ADMIN_ENV%\\bin": + "PATH=%PATH%" + File.pathSeparator + "%TEZ_ADMIN_ENV%\\bin" : "LD_LIBRARY_PATH=$TEZ_ADMIN_ENV_TEST/lib/native"; conf.set(MRJobConfig.MAPRED_ADMIN_USER_ENV, mapredAdminUserEnv); @@ -212,5 +212,4 @@ public void testTranslateMRConfToTez() { Assert.assertNull(conf3.get(org.apache.tez.mapreduce.hadoop.MRJobConfig.IO_SORT_MB)); Assert.assertEquals(500, conf3.getLong(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 0)); } - } diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRInputHelpers.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRInputHelpers.java index 11b1271a8a..8c20bee83a 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRInputHelpers.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRInputHelpers.java @@ -73,7 +73,6 @@ public static void setup() throws IOException { Configuration testConf = new Configuration( dfsCluster.getFileSystem().getConf()); - FSDataOutputStream dataOutputStream = null; try { dataOutputStream = remoteFs.create(new Path("/tmp/input/test.xml"), true); @@ -100,7 +99,6 @@ public static void setup() throws IOException { newSplitsDir = remoteFs.makeQualified(new Path("/tmp/splitsDirNew/")); } - @Test(timeout = 5000) public void testNewSplitsGen() throws Exception { @@ -218,7 +216,6 @@ private DataSourceDescriptor generateDataSourceDescriptorMapReduce(Path inputSpl return MRInputHelpers.configureMRInputWithLegacySplitGeneration(jobConf, inputSplitsDir, true); } - private DataSourceDescriptor generateDataSourceDescriptorMapRed(Path inputSplitsDir) throws Exception { JobConf jobConf = new JobConf(dfsCluster.getFileSystem().getConf()); @@ -257,5 +254,4 @@ public void testInputSplitLocalResourceCreationWithDifferentFS() throws Exceptio localFs.delete(LOCAL_TEST_ROOT_DIR, true); } } - } diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/MRInputForTest.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/MRInputForTest.java index 0d1d24ff6f..a6adacb863 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/MRInputForTest.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/MRInputForTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/MultiMRInputForTest.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/MultiMRInputForTest.java index f0f0a77aa7..786de13aaa 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/MultiMRInputForTest.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/MultiMRInputForTest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/TestMRInput.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/TestMRInput.java index 844ea51cf1..54207f4691 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/TestMRInput.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/TestMRInput.java @@ -72,7 +72,6 @@ public void test0PhysicalInputs() throws IOException { doReturn(new TezCounters()).when(inputContext).getCounters(); doReturn(new JobConf(false)).when(inputContext).getContainerConfiguration(); - MRInput mrInput = new MRInput(inputContext, 0); mrInput.initialize(); @@ -124,14 +123,12 @@ public void testAttributesInJobConf() throws Exception { doReturn(TEST_ATTRIBUTES_UNIQUE_IDENTIFIER).when(inputContext).getUniqueIdentifier(); doReturn(new Configuration(false)).when(inputContext).getContainerConfiguration(); - DataSourceDescriptor dsd = MRInput.createConfigBuilder(new Configuration(false), TestInputFormat.class).groupSplits(false).build(); doReturn(dsd.getInputDescriptor().getUserPayload()).when(inputContext).getUserPayload(); doReturn(new TezCounters()).when(inputContext).getCounters(); - MRInput mrInput = new MRInput(inputContext, 1); mrInput.initialize(); diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/TestMultiMRInput.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/TestMultiMRInput.java index bd6e891bd2..acd8da5342 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/TestMultiMRInput.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/TestMultiMRInput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,6 +19,7 @@ package org.apache.tez.mapreduce.input; import java.nio.ByteBuffer; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -249,7 +250,7 @@ public void testMultipleSplits() throws Exception { } private void assertReaders(MultiMRInput input, LinkedHashMap data, - int expectedReaderCounts, long inputBytes) throws Exception { + int expectedReaderCounts, long inputBytes) throws Exception { int readerCount = 0; int recordCount = 0; for (KeyValueReader reader : input.getKeyValueReaders()) { @@ -267,7 +268,7 @@ private void assertReaders(MultiMRInput input, LinkedHashMap try { reader.next(); //should throw exception fail(); - } catch(IOException e) { + } catch (IOException e) { assertTrue(e.getMessage().contains("For usage, please refer to")); } } @@ -317,7 +318,7 @@ public void testExtraEvents() throws Exception { } private LinkedHashMap createSplits(int splitCount, Path workDir, - Configuration conf, AtomicLong totalSize) throws Exception { + Configuration conf, AtomicLong totalSize) throws Exception { LinkedHashMap data = new LinkedHashMap(); for (int i = 0; i < splitCount; ++i) { int start = i * 10; @@ -359,8 +360,8 @@ public static void cleanUp() throws IOException { } public static LinkedHashMap createInputData(FileSystem fs, Path workDir, - Configuration job, String filename, long startKey, long numKeys, AtomicLong fileLength) - throws IOException { + Configuration job, String filename, long startKey, long numKeys, AtomicLong fileLength) + throws IOException { LinkedHashMap data = new LinkedHashMap(); Path file = new Path(workDir, filename); LOG.info("Generating data at path: " + file); diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/lib/TestKVReadersWithMR.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/lib/TestKVReadersWithMR.java index 21a92469e2..76f9bb4c91 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/lib/TestKVReadersWithMR.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/lib/TestKVReadersWithMR.java @@ -86,7 +86,6 @@ public void testWithSpecificNumberOfKV(int kvPairs) throws IOException { } catch (IOException e) { assertTrue(e.getMessage().contains("For usage, please refer to")); } - } public void testWithSpecificNumberOfKV_MapReduce(int kvPairs) throws IOException { @@ -227,5 +226,4 @@ public float getProgress() throws IOException, InterruptedException { public void close() throws IOException { } } - } diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMROutput.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMROutput.java index bfc09dc9b8..f9158f16c3 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMROutput.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMROutput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -74,13 +74,12 @@ import com.google.common.collect.HashMultimap; import com.google.common.collect.Lists; - public class TestMROutput { static File tmpDir; @BeforeClass - public static void setupClass () { + public static void setupClass() { tmpDir = Files.createTempDir(); tmpDir.deleteOnExit(); } @@ -215,9 +214,9 @@ public void testNewAPI_WorkOutputPathOutputFormat() throws Exception { Configuration conf = new Configuration(); conf.setBoolean(MRConfig.IS_MAP_PROCESSOR, true); DataSinkDescriptor dataSink = MROutput - .createConfigBuilder(conf, NewAPI_WorkOutputPathReadingOutputFormat.class, - tmpDir.getPath()) - .build(); + .createConfigBuilder(conf, NewAPI_WorkOutputPathReadingOutputFormat.class, + tmpDir.getPath()) + .build(); OutputContext outputContext = createMockOutputContext(dataSink.getOutputDescriptor().getUserPayload(), new Configuration(false)); @@ -242,9 +241,9 @@ public void testOldAPI_WorkOutputPathOutputFormat() throws Exception { Configuration conf = new Configuration(); conf.setBoolean(MRConfig.IS_MAP_PROCESSOR, false); DataSinkDescriptor dataSink = MROutput - .createConfigBuilder(conf, OldAPI_WorkOutputPathReadingOutputFormat.class, - tmpDir.getPath()) - .build(); + .createConfigBuilder(conf, OldAPI_WorkOutputPathReadingOutputFormat.class, + tmpDir.getPath()) + .build(); OutputContext outputContext = createMockOutputContext(dataSink.getOutputDescriptor().getUserPayload(), new Configuration(false)); @@ -273,7 +272,7 @@ private OutputContext createMockOutputContext(UserPayload payload, Configuration when(outputContext.getContainerConfiguration()).thenReturn(baseConf); return outputContext; } - + public static LogicalIOProcessorRuntimeTask createLogicalTask( Configuration conf, TezUmbilical umbilical, String dagName, @@ -283,7 +282,7 @@ public static LogicalIOProcessorRuntimeTask createLogicalTask( List outputSpecs = Lists.newLinkedList(); outputSpecs.add(new OutputSpec("Null", MROutput.createConfigBuilder(conf, TestOutputFormat.class).build().getOutputDescriptor(), 1)); - + TaskSpec taskSpec = new TaskSpec( TezTestUtils.getMockTaskAttemptId(0, 0, 0, 0), dagName, vertexName, -1, @@ -294,13 +293,13 @@ public static LogicalIOProcessorRuntimeTask createLogicalTask( FileSystem fs = FileSystem.getLocal(conf); Path workDir = new Path(new Path(System.getProperty("test.build.data", "/tmp")), - "TestMapOutput").makeQualified(fs.getUri(), fs.getWorkingDirectory()); + "TestMapOutput").makeQualified(fs.getUri(), fs.getWorkingDirectory()); return new LogicalIOProcessorRuntimeTask( taskSpec, 0, conf, - new String[] {workDir.toString()}, + new String[]{workDir.toString()}, umbilical, null, new HashMap(), @@ -330,13 +329,13 @@ public void commitTask(TaskAttemptContext taskContext) throws IOException { @Override public void abortTask(TaskAttemptContext taskContext) throws IOException { } - } - + public static class TestOutputFormat extends OutputFormat { public static class TestRecordWriter extends RecordWriter { Writer writer; boolean doWrite; + TestRecordWriter(boolean write) throws IOException { this.doWrite = write; if (doWrite) { @@ -345,7 +344,7 @@ public static class TestRecordWriter extends RecordWriter { writer = new BufferedWriter(new FileWriter(f)); } } - + @Override public void write(String key, String value) throws IOException, InterruptedException { if (doWrite) { @@ -358,9 +357,8 @@ public void write(String key, String value) throws IOException, InterruptedExcep public void close(TaskAttemptContext context) throws IOException, InterruptedException { writer.close(); } - } - + @Override public RecordWriter getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException { @@ -390,7 +388,7 @@ public void close(Reporter reporter) throws IOException {} @Override public org.apache.hadoop.mapred.RecordWriter getRecordWriter( - FileSystem ignored, JobConf job, String name, Progressable progress) throws IOException { + FileSystem ignored, JobConf job, String name, Progressable progress) throws IOException { // check work output path is not null Path workOutputPath = org.apache.hadoop.mapred.FileOutputFormat.getWorkOutputPath(job); assertNotNull(workOutputPath); @@ -427,11 +425,10 @@ public TestProcessor(ProcessorContext context) { @Override public void run() throws Exception { KeyValueWriter writer = (KeyValueWriter) getOutputs().values().iterator().next().getWriter(); - for (int i=0; i<1000000; ++i) { + for (int i = 0; i < 1000000; ++i) { writer.write("key", "value"); } } - } @Ignore diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMROutputConfigBuilder.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMROutputConfigBuilder.java index b54ec3a848..afadd7cd7c 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMROutputConfigBuilder.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMROutputConfigBuilder.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,7 +26,6 @@ import static org.junit.Assert.*; - public class TestMROutputConfigBuilder { @Test(timeout = 5000) @@ -38,7 +37,7 @@ public void testNewAPI() { } catch (TezUncheckedException e) { assertEquals("OutputPaths must be specified for OutputFormats based " + "on org.apache.hadoop.mapreduce.lib.output.FileOutputFormat " - +"or org.apache.hadoop.mapred.FileOutputFormat", e.getMessage()); + + "or org.apache.hadoop.mapred.FileOutputFormat", e.getMessage()); } MROutput.createConfigBuilder(conf, TextOutputFormat.class, "/tmp/output").build(); @@ -53,7 +52,7 @@ public void testNewAPI_ThroughConf() { MROutput.createConfigBuilder(conf, null).build(); fail(); } catch (TezUncheckedException e) { - assertEquals("no outputFormat setting on Configuration, useNewAPI:true", e.getMessage()); + assertEquals("no outputFormat setting on Configuration, useNewAPI:true", e.getMessage()); } // wrong output_format class @@ -76,14 +75,14 @@ public void testNewAPI_ThroughConf() { } catch (TezUncheckedException e) { assertEquals("OutputPaths must be specified for OutputFormats based " + "on org.apache.hadoop.mapreduce.lib.output.FileOutputFormat " - +"or org.apache.hadoop.mapred.FileOutputFormat", e.getMessage()); + + "or org.apache.hadoop.mapred.FileOutputFormat", e.getMessage()); } conf.set(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.OUTDIR, "/tmp/output"); MROutput.createConfigBuilder(conf, null).build(); } - @Test(timeout = 5000 ) + @Test(timeout = 5000) public void testOldAPI() { Configuration conf = new Configuration(); try { @@ -92,7 +91,7 @@ public void testOldAPI() { } catch (TezUncheckedException e) { assertEquals("OutputPaths must be specified for OutputFormats based " + "on org.apache.hadoop.mapreduce.lib.output.FileOutputFormat " - +"or org.apache.hadoop.mapred.FileOutputFormat", e.getMessage()); + + "or org.apache.hadoop.mapred.FileOutputFormat", e.getMessage()); } MROutput.createConfigBuilder(conf, org.apache.hadoop.mapred.TextOutputFormat.class, "/tmp/output").build(); @@ -109,7 +108,7 @@ public void testOldAPI_ThroughConf() { MROutput.createConfigBuilder(conf, null).build(); fail(); } catch (TezUncheckedException e) { - assertEquals("no outputFormat setting on Configuration, useNewAPI:false", e.getMessage()); + assertEquals("no outputFormat setting on Configuration, useNewAPI:false", e.getMessage()); } // wrong output_format class @@ -132,7 +131,7 @@ public void testOldAPI_ThroughConf() { } catch (TezUncheckedException e) { assertEquals("OutputPaths must be specified for OutputFormats based " + "on org.apache.hadoop.mapreduce.lib.output.FileOutputFormat " - +"or org.apache.hadoop.mapred.FileOutputFormat", e.getMessage()); + + "or org.apache.hadoop.mapred.FileOutputFormat", e.getMessage()); } conf.set(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.OUTDIR, "/tmp/output"); diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMROutputLegacy.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMROutputLegacy.java index 60596be89d..efd9c02549 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMROutputLegacy.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMROutputLegacy.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -50,7 +50,7 @@ public class TestMROutputLegacy { TestMROutputLegacy.class.getName()).getAbsoluteFile(); // simulate the behavior of translating MR to DAG using MR old API - @Test (timeout = 5000) + @Test(timeout = 5000) public void testOldAPI_MR() throws Exception { String outputPath = TEST_DIR.getAbsolutePath(); JobConf conf = new JobConf(); @@ -81,7 +81,7 @@ public void testOldAPI_MR() throws Exception { } // simulate the behavior of translating MR to DAG using MR new API - @Test (timeout = 5000) + @Test(timeout = 5000) public void testNewAPI_MR() throws Exception { String outputPath = TEST_DIR.getAbsolutePath(); Job job = Job.getInstance(); @@ -113,7 +113,7 @@ public void testNewAPI_MR() throws Exception { } // simulate the behavior of translating Mapper-only job to DAG using MR old API - @Test (timeout = 5000) + @Test(timeout = 5000) public void testOldAPI_MapperOnly() throws Exception { String outputPath = TEST_DIR.getAbsolutePath(); JobConf conf = new JobConf(); @@ -144,7 +144,7 @@ public void testOldAPI_MapperOnly() throws Exception { } //simulate the behavior of translating mapper-only job to DAG using MR new API - @Test (timeout = 5000) + @Test(timeout = 5000) public void testNewAPI_MapperOnly() throws Exception { String outputPath = TEST_DIR.getAbsolutePath(); Job job = Job.getInstance(); diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMultiMROutput.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMultiMROutput.java index 2662827678..93cb1c0815 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMultiMROutput.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/output/TestMultiMROutput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -43,7 +43,6 @@ import java.io.File; import java.io.IOException; - public class TestMultiMROutput { private static final File TEST_DIR = new File(System.getProperty("test.build.data"), TestMultiMROutput.class.getName()).getAbsoluteFile(); @@ -151,8 +150,8 @@ private OutputContext createMockOutputContext(UserPayload payload) { } private void validate(boolean expectedUseNewAPIValue, Class outputFormat, - boolean isMapper, Class committerClass, boolean useLazyOutputFormat) - throws InterruptedException, IOException { + boolean isMapper, Class committerClass, boolean useLazyOutputFormat) + throws InterruptedException, IOException { MultiMROutput output = createMROutputs(outputFormat, isMapper, useLazyOutputFormat); @@ -194,7 +193,7 @@ private void validate(boolean expectedUseNewAPIValue, Class outputFormat, assertEquals(committerClass, output.committer.getClass()); int numOfUniqueKeys = 3; - for (int i=0; i + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.mapreduce.processor; @@ -70,11 +70,10 @@ import com.google.common.collect.HashMultimap; - public class MapUtils { private static final Logger LOG = LoggerFactory.getLogger(MapUtils.class); - + public static void configureLocalDirs(Configuration conf, String localDir) throws IOException { String[] localSysDirs = new String[1]; @@ -118,8 +117,8 @@ public static void configureLocalDirs(Configuration conf, String localDir) } conf.set(MRFrameworkConfigs.JOB_LOCAL_DIR, workDir.toString()); } - - private static InputSplit + + private static InputSplit createInputSplit(FileSystem fs, Path workDir, JobConf job, Path file, int numKVs) throws IOException { FileInputFormat.setInputPaths(job, workDir); @@ -127,8 +126,8 @@ public static void configureLocalDirs(Configuration conf, String localDir) LOG.info("Generating data at path: " + file); // create a file with length entries @SuppressWarnings("deprecation") - SequenceFile.Writer writer = - SequenceFile.createWriter(fs, job, file, + SequenceFile.Writer writer = + SequenceFile.createWriter(fs, job, file, LongWritable.class, Text.class); try { Random r = new Random(System.currentTimeMillis()); @@ -143,25 +142,25 @@ public static void configureLocalDirs(Configuration conf, String localDir) } finally { writer.close(); } - - SequenceFileInputFormat format = + + SequenceFileInputFormat format = new SequenceFileInputFormat(); InputSplit[] splits = format.getSplits(job, 1); System.err.println("#split = " + splits.length + " ; " + "#locs = " + splits[0].getLocations().length + "; " + - "loc = " + splits[0].getLocations()[0] + "; " + + "loc = " + splits[0].getLocations()[0] + "; " + "off = " + splits[0].getLength() + "; " + - "file = " + ((FileSplit)splits[0]).getPath()); + "file = " + ((FileSplit) splits[0]).getPath()); return splits[0]; } - + final private static FsPermission JOB_FILE_PERMISSION = FsPermission .createImmutable((short) 0644); // rw-r--r-- // Will write files to PWD, from where they are read. - + private static void writeSplitFiles(FileSystem fs, JobConf conf, - InputSplit split) throws IOException { + InputSplit split) throws IOException { Path jobSplitFile = new Path(conf.get(MRFrameworkConfigs.TASK_LOCAL_RESOURCE_DIR, MRFrameworkConfigs.TASK_LOCAL_RESOURCE_DIR_DEFAULT), MRJobConfig.JOB_SPLIT); LOG.info("Writing split to: " + jobSplitFile); @@ -197,18 +196,18 @@ public static void generateInputSplit(FileSystem fs, Path workDir, JobConf jobCo InputSplit split = createInputSplit(fs, workDir, jobConf, mapInput, numKVs); writeSplitFiles(fs, jobConf, split); } - + public static LogicalIOProcessorRuntimeTask createLogicalTask(FileSystem fs, Path workDir, - JobConf jobConf, int mapId, Path mapInput, - TezUmbilical umbilical, String dagName, - String vertexName, List inputSpecs, - List outputSpecs, TezSharedExecutor sharedExecutor) throws Exception { + JobConf jobConf, int mapId, Path mapInput, + TezUmbilical umbilical, String dagName, + String vertexName, List inputSpecs, + List outputSpecs, TezSharedExecutor sharedExecutor) throws Exception { jobConf.setInputFormat(SequenceFileInputFormat.class); ProcessorDescriptor mapProcessorDesc = ProcessorDescriptor.create( MapProcessor.class.getName()).setUserPayload( TezUtils.createUserPayloadFromConf(jobConf)); - + Token shuffleToken = new Token(); TaskSpec taskSpec = new TaskSpec( @@ -233,7 +232,7 @@ public static LogicalIOProcessorRuntimeTask createLogicalTask(FileSystem fs, Pat taskSpec, 0, jobConf, - new String[] {workDir.toString()}, + new String[]{workDir.toString()}, umbilical, serviceConsumerMetadata, envMap, diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java index bd83dfa26c..b389723e6a 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,7 +17,6 @@ */ package org.apache.tez.mapreduce.processor.map; - import java.io.IOException; import java.nio.ByteBuffer; import java.util.Collections; @@ -68,30 +67,27 @@ @SuppressWarnings("deprecation") public class TestMapProcessor { - + private static final Logger LOG = LoggerFactory.getLogger(TestMapProcessor.class); - + private static JobConf defaultConf = new JobConf(); - private static FileSystem localFs = null; + private static FileSystem localFs = null; private static Path workDir = null; static float progressUpdate = 0.0f; + static { try { defaultConf.set("fs.defaultFS", "file:///"); localFs = FileSystem.getLocal(defaultConf); workDir = new Path(new Path(System.getProperty("test.build.data", "/tmp")), - "TestMapProcessor").makeQualified(localFs); + "TestMapProcessor").makeQualified(localFs); LOG.info("Using workDir: " + workDir); MapUtils.configureLocalDirs(defaultConf, workDir.toString()); } catch (IOException e) { throw new RuntimeException("init failure", e); } } - - - - public void setUpJobConf(JobConf job) { job.set(TezRuntimeFrameworkConfigs.LOCAL_DIRS, workDir.toString()); @@ -110,7 +106,7 @@ private Path getMapOutputFile(Configuration jobConf, OutputContext outputContext Path attemptOutput = new Path(new Path(Constants.TEZ_RUNTIME_TASK_OUTPUT_DIR, outputContext.getUniqueIdentifier()), Constants.TEZ_RUNTIME_TASK_OUTPUT_FILENAME_STRING); Path mapOutputFile = lDirAlloc.getLocalPathToRead(attemptOutput.toString(), jobConf); - return mapOutputFile; + return mapOutputFile; } @Before @@ -118,7 +114,7 @@ private Path getMapOutputFile(Configuration jobConf, OutputContext outputContext public void cleanup() throws Exception { localFs.delete(workDir, true); } - + @Test(timeout = 5000) public void testMapProcessor() throws Exception { String dagName = "mrdag0"; @@ -133,10 +129,9 @@ public void testMapProcessor() throws Exception { jobConf.set(MRFrameworkConfigs.TASK_LOCAL_RESOURCE_DIR, new Path(workDir, "localized-resources").toUri().toString()); - + Path mapInput = new Path(workDir, "map0"); - - + MapUtils.generateInputSplit(localFs, workDir, jobConf, mapInput, 10); InputSpec mapInputSpec = new InputSpec("NullSrcVertex", @@ -146,7 +141,7 @@ public void testMapProcessor() throws Exception { .setConfigurationBytes(TezUtils.createByteStringFromConf(jobConf)).build() .toByteArray()))), 1); - OutputSpec mapOutputSpec = new OutputSpec("NullDestVertex", + OutputSpec mapOutputSpec = new OutputSpec("NullDestVertex", OutputDescriptor.create(OrderedPartitionedKVOutput.class.getName()) .setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1); @@ -165,8 +160,7 @@ public void testMapProcessor() throws Exception { TezTaskOutput mapOutputs = new TezTaskOutputFiles( jobConf, outputContext.getUniqueIdentifier(), outputContext.getDagIdentifier()); - - + // TODO NEWTEZ FIXME OutputCommitter verification // MRTask mrTask = (MRTask)t.getProcessor(); // Assert.assertEquals(TezNullOutputCommitter.class.getName(), mrTask @@ -187,7 +181,7 @@ public void testMapProcessor() throws Exception { key.readFields(keyBuf); value.readFields(valueBuf); if (prev != Long.MIN_VALUE) { - assert(prev <= key.get()); + assert (prev <= key.get()); prev = key.get(); } LOG.info("key = " + key.get() + "; value = " + value); @@ -212,7 +206,6 @@ public void testMapProcessorProgress() throws Exception { Path mapInput = new Path(workDir, "map0"); - MapUtils.generateInputSplit(localFs, workDir, jobConf, mapInput, 100000); InputSpec mapInputSpec = new InputSpec("NullSrcVertex", @@ -239,7 +232,7 @@ public void testMapProcessorProgress() throws Exception { @Override public void run() { float prog = task.getProgress(); - if(prog > 0.0f && prog < 1.0f) + if (prog > 0.0f && prog < 1.0f) progressUpdate = prog; } }); diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java index 0223482da6..4eb515f0e3 100644 --- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java +++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,7 +17,6 @@ */ package org.apache.tez.mapreduce.processor.reduce; - import java.io.IOException; import java.nio.ByteBuffer; import java.util.Collections; @@ -85,22 +84,22 @@ import com.google.common.collect.HashMultimap; - @SuppressWarnings("deprecation") public class TestReduceProcessor { - + private static final Logger LOG = LoggerFactory.getLogger(TestReduceProcessor.class); private static JobConf defaultConf = new JobConf(); private static FileSystem localFs = null; private static Path workDir = null; + static { try { defaultConf.set("fs.defaultFS", "file:///"); localFs = FileSystem.getLocal(defaultConf); workDir = new Path(new Path(System.getProperty("test.build.data", "/tmp")), - "TestReduceProcessor").makeQualified(localFs); + "TestReduceProcessor").makeQualified(localFs); LOG.info("Using workDir: " + workDir); MapUtils.configureLocalDirs(defaultConf, workDir.toString()); } catch (IOException e) { @@ -133,14 +132,14 @@ public void testReduceProcessor() throws Exception { String reduceVertexName = MultiStageMRConfigUtil.getFinalReduceVertexName(); JobConf jobConf = new JobConf(defaultConf); setUpJobConf(jobConf); - + MRHelpers.translateMRConfToTez(jobConf); jobConf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, 0); jobConf.set(MRFrameworkConfigs.TASK_LOCAL_RESOURCE_DIR, new Path(workDir, "localized-resources").toUri().toString()); jobConf.setBoolean(MRJobConfig.MR_TEZ_SPLITS_VIA_EVENTS, false); - + Path mapInput = new Path(workDir, "map0"); MapUtils.generateInputSplit(localFs, workDir, jobConf, mapInput, 10); @@ -151,9 +150,9 @@ public void testReduceProcessor() throws Exception { .setConfigurationBytes(TezUtils.createByteStringFromConf(jobConf)).build() .toByteArray()))), 1); - OutputSpec mapOutputSpec = new OutputSpec("NullDestVertex", + OutputSpec mapOutputSpec = new OutputSpec("NullDestVertex", OutputDescriptor.create(OrderedPartitionedKVOutput.class.getName()). - setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1); + setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1); // Run a map TestUmbilical testUmbilical = new TestUmbilical(); @@ -177,9 +176,9 @@ public void testReduceProcessor() throws Exception { Assert.assertEquals(1, cdmEvent.getCount()); DataMovementEvent dme = cdmEvent.getEvents().iterator().next(); dme.setTargetIndex(0); - + LOG.info("Starting reduce..."); - + JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(dagName)); JobTokenSecretManager jobTokenSecretManager = new JobTokenSecretManager(); Token shuffleToken = new Token(identifier, @@ -194,7 +193,7 @@ public void testReduceProcessor() throws Exception { ProcessorDescriptor reduceProcessorDesc = ProcessorDescriptor.create( ReduceProcessor.class.getName()).setUserPayload( TezUtils.createUserPayloadFromConf(jobConf)); - + InputSpec reduceInputSpec = new InputSpec(mapVertexName, InputDescriptor.create(OrderedGroupedInputLegacy.class.getName()) .setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1); @@ -226,7 +225,7 @@ public void testReduceProcessor() throws Exception { taskSpec, 0, jobConf, - new String[] {workDir.toString()}, + new String[]{workDir.toString()}, new TestUmbilical(), serviceConsumerMetadata, serviceProviderEnvMap, @@ -248,8 +247,6 @@ public void testReduceProcessor() throws Exception { // Likely not applicable anymore. // Assert.assertNull(mrTask.getPartitioner()); - - // Only a task commit happens, hence the data is still in the temporary directory. Path reduceOutputDir = new Path(new Path(workDir, "output"), "_temporary/0/" + IDConverter @@ -272,5 +269,4 @@ public void testReduceProcessor() throws Exception { reader.close(); } - } diff --git a/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/FadvisedChunkedFile.java b/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/FadvisedChunkedFile.java index 162feb9801..3a3cf7a827 100644 --- a/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/FadvisedChunkedFile.java +++ b/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/FadvisedChunkedFile.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/FadvisedFileRegion.java b/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/FadvisedFileRegion.java index 23663636e5..31af01e0f3 100644 --- a/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/FadvisedFileRegion.java +++ b/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/FadvisedFileRegion.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -83,7 +83,7 @@ public long transferTo(WritableByteChannel target, long position) } long written = 0; - if(this.shuffleTransferToAllowed) { + if (this.shuffleTransferToAllowed) { written = super.transferTo(target, position); } else { written = customShuffleTransfer(target, position); @@ -140,10 +140,10 @@ long customShuffleTransfer(WritableByteChannel target, long position) int readSize; ByteBuffer byteBuffer = ByteBuffer.allocate(this.shuffleBufferSize); - while(trans > 0L && - (readSize = fileChannel.read(byteBuffer, this.position+position)) > 0) { + while (trans > 0L && + (readSize = fileChannel.read(byteBuffer, this.position + position)) > 0) { //adjust counters and buffer limit - if(readSize < trans) { + if (readSize < trans) { trans -= readSize; position += readSize; byteBuffer.flip(); @@ -151,14 +151,14 @@ long customShuffleTransfer(WritableByteChannel target, long position) //We can read more than we need if the actualCount is not multiple //of the byteBuffer size and file is big enough. In that case we cannot //use flip method but we need to set buffer limit manually to trans. - byteBuffer.limit((int)trans); + byteBuffer.limit((int) trans); byteBuffer.position(0); position += trans; trans = 0; } //write data to the target - while(byteBuffer.hasRemaining()) { + while (byteBuffer.hasRemaining()) { target.write(byteBuffer); } diff --git a/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/IndexCache.java b/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/IndexCache.java index 54db975292..9cf539799c 100644 --- a/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/IndexCache.java +++ b/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/IndexCache.java @@ -34,8 +34,8 @@ class IndexCache { private AtomicInteger totalMemoryUsed = new AtomicInteger(); private static final Logger LOG = LoggerFactory.getLogger(IndexCache.class); - private final ConcurrentHashMap cache = - new ConcurrentHashMap(); + private final ConcurrentHashMap cache = + new ConcurrentHashMap(); private final LinkedBlockingQueue queue = new LinkedBlockingQueue(); @@ -60,9 +60,10 @@ private void initLocalFs() { /** * This method gets the spill record for the given mapId. * It reads the index file into cache if it is not already present. + * * @param mapId - * @param fileName The file to read the index information from if it is not - * already present in the cache + * @param fileName The file to read the index information from if it is not + * already present in the cache * @param expectedIndexOwner The expected owner of the index file * @return The spill record for this map * @throws IOException @@ -75,7 +76,7 @@ public TezSpillRecord getSpillRecord(String mapId, Path fileName, String expecte if (info == null) { info = readIndexFileToCache(fileName, mapId, expectedIndexOwner); } else { - synchronized(info) { + synchronized (info) { while (isUnderConstruction(info)) { try { info.wait(); @@ -97,16 +98,17 @@ public TezSpillRecord getSpillRecord(String mapId, Path fileName, String expecte /** * This method gets the index information for the given mapId and reduce. * It reads the index file into cache if it is not already present. + * * @param mapId * @param reduce - * @param fileName The file to read the index information from if it is not - * already present in the cache + * @param fileName The file to read the index information from if it is not + * already present in the cache * @param expectedIndexOwner The expected owner of the index file * @return The Index Information * @throws IOException */ public TezIndexRecord getIndexInformation(String mapId, int reduce, - Path fileName, String expectedIndexOwner) + Path fileName, String expectedIndexOwner) throws IOException { IndexInformation info = cache.get(mapId); @@ -114,7 +116,7 @@ public TezIndexRecord getIndexInformation(String mapId, int reduce, if (info == null) { info = readIndexFileToCache(fileName, mapId, expectedIndexOwner); } else { - synchronized(info) { + synchronized (info) { while (isUnderConstruction(info)) { try { info.wait(); @@ -136,7 +138,7 @@ public TezIndexRecord getIndexInformation(String mapId, int reduce, } private boolean isUnderConstruction(IndexInformation info) { - synchronized(info) { + synchronized (info) { return (null == info.mapSpillRecord); } } @@ -148,7 +150,7 @@ private IndexInformation readIndexFileToCache(Path indexFileName, IndexInformation info; IndexInformation newInd = new IndexInformation(); if ((info = cache.putIfAbsent(mapId, newInd)) != null) { - synchronized(info) { + synchronized (info) { while (isUnderConstruction(info)) { try { info.wait(); @@ -189,6 +191,7 @@ private IndexInformation readIndexFileToCache(Path indexFileName, * removed if it is in the loading phrase(size=0), this prevents corruption * of totalMemoryUsed. It should be called when a map output on this tracker * is discarded. + * * @param mapId The taskID of this map. */ public void removeMap(String mapId) { @@ -210,6 +213,7 @@ public void removeMap(String mapId) { /** * This method checks if cache and totolMemoryUsed is consistent. * It is only used for unit test. + * * @return True if cache and totolMemoryUsed is consistent */ boolean checkTotalMemoryUsed() { diff --git a/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java b/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java index 0fa1c03a96..777d41256f 100644 --- a/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java +++ b/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java @@ -1,25 +1,26 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.auxservices; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.util.DiskChecker; + import static org.fusesource.leveldbjni.JniDBFactory.asString; import static org.fusesource.leveldbjni.JniDBFactory.bytes; import static io.netty.buffer.Unpooled.wrappedBuffer; @@ -161,7 +162,7 @@ public class ShuffleHandler extends AuxiliaryService { private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(ShuffleHandler.class); private static final org.slf4j.Logger AUDITLOG = - LoggerFactory.getLogger(ShuffleHandler.class.getName()+".audit"); + LoggerFactory.getLogger(ShuffleHandler.class.getName() + ".audit"); public static final String SHUFFLE_MANAGE_OS_CACHE = "tez.shuffle.manage.os.cache"; public static final boolean DEFAULT_SHUFFLE_MANAGE_OS_CACHE = true; @@ -206,7 +207,7 @@ public class ShuffleHandler extends AuxiliaryService { private int maxSessionOpenFiles; private ReadaheadPool readaheadPool = ReadaheadPool.getInstance(); - private Map userRsrc; + private Map userRsrc; private JobTokenSecretManager secretManager; private DB stateDb = null; @@ -236,7 +237,7 @@ public class ShuffleHandler extends AuxiliaryService { public static final boolean SHUFFLE_SSL_ENABLED_DEFAULT = false; public static final String SUFFLE_SSL_FILE_BUFFER_SIZE_KEY = - "tez.shuffle.ssl.file.buffer.size"; + "tez.shuffle.ssl.file.buffer.size"; public static final int DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE = 60 * 1024; @@ -251,7 +252,7 @@ public class ShuffleHandler extends AuxiliaryService { "tez.shuffle.transfer.buffer.size"; public static final int DEFAULT_SHUFFLE_BUFFER_SIZE = 128 * 1024; - public static final String SHUFFLE_TRANSFERTO_ALLOWED = + public static final String SHUFFLE_TRANSFERTO_ALLOWED = "tez.shuffle.transferTo.allowed"; public static final boolean DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED = true; public static final boolean WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED = @@ -270,16 +271,16 @@ public class ShuffleHandler extends AuxiliaryService { private int connectionKeepAliveTimeOut; private int mapOutputMetaInfoCacheSize; - @Metrics(about="Shuffle output metrics", context="mapred", name="tez") + @Metrics(about = "Shuffle output metrics", context = "mapred", name = "tez") static class ShuffleMetrics implements ChannelFutureListener { @Metric("Shuffle output in bytes") - MutableCounterLong shuffleOutputBytes; + MutableCounterLong shuffleOutputBytes; @Metric("# of failed shuffle outputs") - MutableCounterInt shuffleOutputsFailed; + MutableCounterInt shuffleOutputsFailed; @Metric("# of succeeeded shuffle outputs") - MutableCounterInt shuffleOutputsOK; + MutableCounterInt shuffleOutputsOK; @Metric("# of current shuffle connections") - MutableGaugeInt shuffleConnections; + MutableGaugeInt shuffleConnections; @Override public void operationComplete(ChannelFuture future) throws Exception { @@ -352,17 +353,17 @@ public ReduceContext(List mapIds, Range reduceRange, this.reduceRange = reduceRange; this.dagId = dagId; /** - * Atomic count for tracking the no. of map outputs that are yet to - * complete. Multiple futureListeners' operationComplete() can decrement - * this value asynchronously. It is used to decide when the channel should - * be closed. - */ + * Atomic count for tracking the no. of map outputs that are yet to + * complete. Multiple futureListeners' operationComplete() can decrement + * this value asynchronously. It is used to decide when the channel should + * be closed. + */ this.mapsToWait = new AtomicInteger(mapIds.size()); /** - * Atomic count for tracking the no. of map outputs that have been sent. - * Multiple sendMap() calls can increment this value - * asynchronously. Used to decide which mapId should be sent next. - */ + * Atomic count for tracking the no. of map outputs that have been sent. + * Multiple sendMap() calls can increment this value + * asynchronously. Used to decide which mapId should be sent next. + */ this.mapsToSend = new AtomicInteger(0); this.ctx = context; this.user = usr; @@ -480,7 +481,7 @@ public void initializeApplication(ApplicationInitializationContext context) { // TODO these bytes should be versioned try { Token jt = deserializeServiceData(secret); - // TODO: Once SHuffle is out of NM, this can use MR APIs + // TODO: Once SHuffle is out of NM, this can use MR APIs JobID jobId = new JobID(Long.toString(appId.getClusterTimestamp()), appId.getId()); recordJobShuffleInfo(jobId, user, jt); } catch (IOException e) { @@ -510,19 +511,19 @@ protected void serviceInit(Configuration conf) throws Exception { DEFAULT_SHUFFLE_READAHEAD_BYTES); maxShuffleConnections = conf.getInt(MAX_SHUFFLE_CONNECTIONS, - DEFAULT_MAX_SHUFFLE_CONNECTIONS); + DEFAULT_MAX_SHUFFLE_CONNECTIONS); int maxShuffleThreads = conf.getInt(MAX_SHUFFLE_THREADS, - DEFAULT_MAX_SHUFFLE_THREADS); + DEFAULT_MAX_SHUFFLE_THREADS); if (maxShuffleThreads == 0) { maxShuffleThreads = 2 * Runtime.getRuntime().availableProcessors(); } shuffleBufferSize = conf.getInt(SHUFFLE_BUFFER_SIZE, - DEFAULT_SHUFFLE_BUFFER_SIZE); + DEFAULT_SHUFFLE_BUFFER_SIZE); shuffleTransferToAllowed = conf.getBoolean(SHUFFLE_TRANSFERTO_ALLOWED, - (Shell.WINDOWS)?WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED: - DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED); + (Shell.WINDOWS) ? WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED : + DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED); maxSessionOpenFiles = conf.getInt(SHUFFLE_MAX_SESSION_OPEN_FILES, DEFAULT_SHUFFLE_MAX_SESSION_OPEN_FILES); @@ -552,7 +553,7 @@ public Thread newThread(Runnable r) { @Override protected void serviceStart() throws Exception { Configuration conf = getConfig(); - userRsrc = new ConcurrentHashMap(); + userRsrc = new ConcurrentHashMap(); secretManager = new JobTokenSecretManager(); recoverState(conf); ServerBootstrap bootstrap = new ServerBootstrap() @@ -567,7 +568,7 @@ protected void serviceStart() throws Exception { accepted.add(ch); // setup port - port = ((InetSocketAddress)ch.localAddress()).getPort(); + port = ((InetSocketAddress) ch.localAddress()).getPort(); conf.set(SHUFFLE_PORT_CONFIG_KEY, Integer.toString(port)); SHUFFLE.setPort(port); LOG.info(getName() + " listening on port " + port); @@ -575,16 +576,16 @@ protected void serviceStart() throws Exception { super.serviceStart(); sslFileBufferSize = conf.getInt(SUFFLE_SSL_FILE_BUFFER_SIZE_KEY, - DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE); + DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE); connectionKeepAliveEnabled = conf.getBoolean(SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED, - DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED); + DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED); connectionKeepAliveTimeOut = Math.max(1, conf.getInt(SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT, - DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT)); + DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT)); mapOutputMetaInfoCacheSize = Math.max(1, conf.getInt(SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE, - DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE)); + DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE)); } private void initPipeline(ServerBootstrap bootstrap, Configuration conf) throws Exception { @@ -598,20 +599,20 @@ private void initPipeline(ServerBootstrap bootstrap, Configuration conf) throws ChannelInitializer channelInitializer = new ChannelInitializer() { @Override - public void initChannel(NioSocketChannel ch) throws Exception { - ChannelPipeline pipeline = ch.pipeline(); - if (sslFactory != null) { - pipeline.addLast("ssl", new SslHandler(sslFactory.createSSLEngine())); - } - pipeline.addLast("decoder", new HttpRequestDecoder()); - pipeline.addLast("aggregator", new HttpObjectAggregator(1 << 16)); - pipeline.addLast("encoder", new HttpResponseEncoder()); - pipeline.addLast("chunking", new ChunkedWriteHandler()); - pipeline.addLast("shuffle", SHUFFLE); - pipeline.addLast("idle", new IdleStateHandler(0, connectionKeepAliveTimeOut, 0)); - pipeline.addLast(TIMEOUT_HANDLER, new TimeoutHandler()); - } - }; + public void initChannel(NioSocketChannel ch) throws Exception { + ChannelPipeline pipeline = ch.pipeline(); + if (sslFactory != null) { + pipeline.addLast("ssl", new SslHandler(sslFactory.createSSLEngine())); + } + pipeline.addLast("decoder", new HttpRequestDecoder()); + pipeline.addLast("aggregator", new HttpObjectAggregator(1 << 16)); + pipeline.addLast("encoder", new HttpResponseEncoder()); + pipeline.addLast("chunking", new ChunkedWriteHandler()); + pipeline.addLast("shuffle", SHUFFLE); + pipeline.addLast("idle", new IdleStateHandler(0, connectionKeepAliveTimeOut, 0)); + pipeline.addLast(TIMEOUT_HANDLER, new TimeoutHandler()); + } + }; bootstrap.childHandler(channelInitializer); } @@ -666,7 +667,7 @@ private void recoverState(Configuration conf) throws IOException { iter = new LeveldbIterator(stateDb); iter.seek(bytes(JobID.JOB)); while (iter.hasNext()) { - Map.Entry entry = iter.next(); + Map.Entry entry = iter.next(); String key = asString(entry.getKey()); if (!jobPattern.matcher(key).matches()) { break; @@ -767,13 +768,13 @@ private void checkVersion() throws IOException { storeVersion(); } else { throw new IOException( - "Incompatible version for state DB schema: expecting DB schema version " - + getCurrentVersion() + ", but loading version " + loadedVersion); + "Incompatible version for state DB schema: expecting DB schema version " + + getCurrentVersion() + ", but loading version " + loadedVersion); } } private void addJobToken(JobID jobId, String user, - Token jobToken) { + Token jobToken) { userRsrc.put(jobId.toString(), user); getSecretManager().addTokenForJob(jobId.toString(), jobToken); LOG.info("Added token for " + jobId.toString()); @@ -799,7 +800,7 @@ private void recoverJobShuffleInfo(String jobIdStr, byte[] data) } private void recordJobShuffleInfo(JobID jobId, String user, - Token jobToken) throws IOException { + Token jobToken) throws IOException { if (stateDb != null) { // Discover type instead of assuming ByteString to allow for shading. TokenProto tokenProto = TokenProto.newBuilder() @@ -893,48 +894,48 @@ class Shuffle extends ChannelInboundHandlerAdapter { private final Configuration conf; private final IndexCache indexCache; private final LocalDirAllocator lDirAlloc = - new LocalDirAllocator(YarnConfiguration.NM_LOCAL_DIRS); + new LocalDirAllocator(YarnConfiguration.NM_LOCAL_DIRS); private int port; private final LoadingCache pathCache = - CacheBuilder.newBuilder().expireAfterAccess(EXPIRE_AFTER_ACCESS_MINUTES, - TimeUnit.MINUTES).softValues().concurrencyLevel(ALLOWED_CONCURRENCY). - removalListener( - new RemovalListener() { - @Override - public void onRemoval(RemovalNotification notification) { - if (LOG.isDebugEnabled()) { - LOG.debug("PathCache Eviction: " + notification.getKey() + - ", Reason=" + notification.getCause()); + CacheBuilder.newBuilder().expireAfterAccess(EXPIRE_AFTER_ACCESS_MINUTES, + TimeUnit.MINUTES).softValues().concurrencyLevel(ALLOWED_CONCURRENCY). + removalListener( + new RemovalListener() { + @Override + public void onRemoval(RemovalNotification notification) { + if (LOG.isDebugEnabled()) { + LOG.debug("PathCache Eviction: " + notification.getKey() + + ", Reason=" + notification.getCause()); + } + } + } + ).maximumWeight(MAX_WEIGHT).weigher( + new Weigher() { + @Override + public int weigh(AttemptPathIdentifier key, + AttemptPathInfo value) { + return key.jobId.length() + key.user.length() + + key.attemptId.length() + + value.indexPath.toString().length() + + value.dataPath.toString().length(); + } + } + ).build(new CacheLoader() { + @Override + public AttemptPathInfo load(AttemptPathIdentifier key) throws + Exception { + String base = getBaseLocation(key.jobId, key.dagId, key.user); + String attemptBase = base + key.attemptId; + Path indexFileName = lDirAlloc.getLocalPathToRead( + attemptBase + Path.SEPARATOR + INDEX_FILE_NAME, conf); + Path mapOutputFileName = lDirAlloc.getLocalPathToRead( + attemptBase + Path.SEPARATOR + DATA_FILE_NAME, conf); + + LOG.debug("Loaded : {} via loader", key); + return new AttemptPathInfo(indexFileName, mapOutputFileName); } - } - } - ).maximumWeight(MAX_WEIGHT).weigher( - new Weigher() { - @Override - public int weigh(AttemptPathIdentifier key, - AttemptPathInfo value) { - return key.jobId.length() + key.user.length() + - key.attemptId.length()+ - value.indexPath.toString().length() + - value.dataPath.toString().length(); - } - } - ).build(new CacheLoader() { - @Override - public AttemptPathInfo load(AttemptPathIdentifier key) throws - Exception { - String base = getBaseLocation(key.jobId, key.dagId, key.user); - String attemptBase = base + key.attemptId; - Path indexFileName = lDirAlloc.getLocalPathToRead( - attemptBase + Path.SEPARATOR + INDEX_FILE_NAME, conf); - Path mapOutputFileName = lDirAlloc.getLocalPathToRead( - attemptBase + Path.SEPARATOR + DATA_FILE_NAME, conf); - - LOG.debug("Loaded : {} via loader", key); - return new AttemptPathInfo(indexFileName, mapOutputFileName); - } - }); + }); public Shuffle(Configuration conf) { this.conf = conf; @@ -976,7 +977,7 @@ public void channelActive(ChannelHandlerContext ctx) if ((maxShuffleConnections > 0) && (accepted.size() >= maxShuffleConnections)) { LOG.info(String.format("Current number of shuffle connections (%d) is " + - "greater than or equal to the max allowed shuffle connections (%d)", + "greater than or equal to the max allowed shuffle connections (%d)", accepted.size(), maxShuffleConnections)); ctx.channel().close(); return; @@ -996,14 +997,14 @@ public void channelRead(ChannelHandlerContext ctx, Object message) private void handleRequest(ChannelHandlerContext ctx, FullHttpRequest request) throws IOException, Exception { if (request.getMethod() != GET) { - sendError(ctx, METHOD_NOT_ALLOWED); - return; + sendError(ctx, METHOD_NOT_ALLOWED); + return; } // Check whether the shuffle version is compatible if (!ShuffleHeader.DEFAULT_HTTP_HEADER_NAME.equals( request.headers().get(ShuffleHeader.HTTP_HEADER_NAME)) || !ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION.equals( - request.headers().get(ShuffleHeader.HTTP_HEADER_VERSION))) { + request.headers().get(ShuffleHeader.HTTP_HEADER_VERSION))) { sendError(ctx, "Incompatible shuffle request version", BAD_REQUEST); return; } @@ -1031,7 +1032,7 @@ private void handleRequest(ChannelHandlerContext ctx, FullHttpRequest request) "\n keepAlive: " + keepAliveParam); } // If the request is for Dag Deletion, process the request and send OK. - if (deleteDagDirectories(ctx.channel(), dagCompletedQ, jobQ, dagIdQ)) { + if (deleteDagDirectories(ctx.channel(), dagCompletedQ, jobQ, dagIdQ)) { return; } if (deleteVertexDirectories(ctx.channel(), vertexCompletedQ, jobQ, dagIdQ, vertexIdQ)) { @@ -1054,7 +1055,7 @@ private void handleRequest(ChannelHandlerContext ctx, FullHttpRequest request) // on log4j.properties by uncommenting the setting if (AUDITLOG.isDebugEnabled()) { AUDITLOG.debug("shuffle for " + jobQ.get(0) + " mapper: " + mapIds - + " reducer: " + reduceRange); + + " reducer: " + reduceRange); } String jobId; String dagId; @@ -1088,13 +1089,13 @@ private void handleRequest(ChannelHandlerContext ctx, FullHttpRequest request) new HashMap(); Channel ch = ctx.channel(); ChannelPipeline pipeline = ch.pipeline(); - TimeoutHandler timeoutHandler = (TimeoutHandler)pipeline.get(TIMEOUT_HANDLER); + TimeoutHandler timeoutHandler = (TimeoutHandler) pipeline.get(TIMEOUT_HANDLER); timeoutHandler.setEnabledTimeout(false); String user = userRsrc.get(jobId); try { populateHeaders(mapIds, jobId, dagId, user, reduceRange, - response, keepAliveParam, mapOutputInfoMap); + response, keepAliveParam, mapOutputInfoMap); } catch (DiskErrorException e) { // fatal error: fetcher should be aware of that LOG.error("Shuffle error in populating headers (fatal: DiskErrorException):", e); String errorMessage = getErrorMessage(e); @@ -1117,7 +1118,7 @@ private void handleRequest(ChannelHandlerContext ctx, FullHttpRequest request) user, mapOutputInfoMap, jobId, dagId, keepAlive); for (int i = 0; i < Math.min(maxSessionOpenFiles, mapIds.size()); i++) { ChannelFuture nextMap = sendMap(reduceContext); - if(nextMap == null) { + if (nextMap == null) { // by this special message flushed, we can make sure the whole response is finished ch.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT); return; @@ -1144,15 +1145,15 @@ private boolean deleteDagDirectories(Channel channel, if (jobQ == null || jobQ.isEmpty()) { return false; } - if (notEmptyAndContains(dagCompletedQ,"delete") && !isNullOrEmpty(dagIdQ)) { + if (notEmptyAndContains(dagCompletedQ, "delete") && !isNullOrEmpty(dagIdQ)) { String base = getDagLocation(jobQ.get(0), dagIdQ.get(0), userRsrc.get(jobQ.get(0))); try { FileContext lfc = FileContext.getLocalFSFileContext(); - for(Path dagPath : lDirAlloc.getAllLocalPathsToRead(base, conf)) { + for (Path dagPath : lDirAlloc.getAllLocalPathsToRead(base, conf)) { lfc.delete(dagPath, true); } } catch (IOException e) { - LOG.warn("Encountered exception during dag delete "+ e); + LOG.warn("Encountered exception during dag delete " + e); } channel.writeAndFlush(new DefaultHttpResponse(HTTP_1_1, OK)) .addListener(ChannelFutureListener.CLOSE); @@ -1174,18 +1175,18 @@ private boolean deleteVertexDirectories(Channel channel, List vertexComp LOG.warn("Encountered exception during vertex delete " + e); } channel.writeAndFlush(new DefaultHttpResponse(HTTP_1_1, OK)) - .addListener(ChannelFutureListener.CLOSE); + .addListener(ChannelFutureListener.CLOSE); return true; } return false; } private boolean deleteTaskAttemptDirectories(Channel channel, List taskAttemptFailedQ, - List jobQ, List dagIdQ, List taskAttemptIdQ) { + List jobQ, List dagIdQ, List taskAttemptIdQ) { if (jobQ == null || jobQ.isEmpty()) { return false; } - if (notEmptyAndContains(taskAttemptFailedQ,"delete") && !isNullOrEmpty(taskAttemptIdQ)) { + if (notEmptyAndContains(taskAttemptFailedQ, "delete") && !isNullOrEmpty(taskAttemptIdQ)) { for (String taskAttemptId : taskAttemptIdQ) { String baseStr = getBaseLocation(jobQ.get(0), dagIdQ.get(0), userRsrc.get(jobQ.get(0))); try { @@ -1208,7 +1209,7 @@ private boolean deleteTaskAttemptDirectories(Channel channel, List taskA } } channel.writeAndFlush(new DefaultHttpResponse(HTTP_1_1, OK)) - .addListener(ChannelFutureListener.CLOSE); + .addListener(ChannelFutureListener.CLOSE); return true; } return false; @@ -1287,7 +1288,7 @@ private String getBaseLocation(String jobId, String dagId, String user) { private void deleteTaskDirsOfVertex(String jobId, String dagId, String vertexId, String user) throws IOException { String baseStr = getBaseLocation(jobId, dagId, user); FileContext lfc = FileContext.getLocalFSFileContext(); - for(Path dagPath : lDirAlloc.getAllLocalPathsToRead(baseStr, conf)) { + for (Path dagPath : lDirAlloc.getAllLocalPathsToRead(baseStr, conf)) { RemoteIterator status = lfc.listStatus(dagPath); final JobID jobID = JobID.forName(jobId); String taskDirPrefix = String.format("attempt%s_%s_%s_", @@ -1296,7 +1297,7 @@ private void deleteTaskDirsOfVertex(String jobId, String dagId, String vertexId, FileStatus fileStatus = status.next(); Path attemptPath = fileStatus.getPath(); if (attemptPath.getName().startsWith(taskDirPrefix)) { - if(lfc.delete(attemptPath, true)) { + if (lfc.delete(attemptPath, true)) { LOG.debug("deleted shuffle data in task directory: {}", attemptPath); } } @@ -1442,7 +1443,7 @@ public void finish() { } protected void verifyRequest(String appid, ChannelHandlerContext ctx, - HttpRequest request, HttpResponse response, URL requestUri) + HttpRequest request, HttpResponse response, URL requestUri) throws IOException { SecretKey tokenSecret = getSecretManager().retrieveTokenSecret(appid); if (null == tokenSecret) { @@ -1453,7 +1454,7 @@ protected void verifyRequest(String appid, ChannelHandlerContext ctx, String enc_str = SecureShuffleUtils.buildMsgFrom(requestUri); // hash from the fetcher String urlHashStr = - request.headers().get(SecureShuffleUtils.HTTP_HEADER_URL_HASH); + request.headers().get(SecureShuffleUtils.HTTP_HEADER_URL_HASH); if (urlHashStr == null) { LOG.info("Missing header hash for " + appid); throw new IOException("fetcher cannot be authenticated"); @@ -1461,14 +1462,14 @@ protected void verifyRequest(String appid, ChannelHandlerContext ctx, if (LOG.isDebugEnabled()) { int len = urlHashStr.length(); LOG.debug("verifying request. enc_str=" + enc_str + "; hash=..." + - urlHashStr.substring(len-len/2, len-1)); + urlHashStr.substring(len - len / 2, len - 1)); } // verify - throws exception SecureShuffleUtils.verifyReply(urlHashStr, enc_str, tokenSecret); // verification passed - encode the reply String reply = - SecureShuffleUtils.generateHash(urlHashStr.getBytes(Charsets.UTF_8), - tokenSecret); + SecureShuffleUtils.generateHash(urlHashStr.getBytes(Charsets.UTF_8), + tokenSecret); response.headers().set(SecureShuffleUtils.HTTP_HEADER_REPLY_URL_HASH, reply); // Put shuffle version into http header response.headers().set(ShuffleHeader.HTTP_HEADER_NAME, @@ -1478,7 +1479,7 @@ protected void verifyRequest(String appid, ChannelHandlerContext ctx, if (LOG.isDebugEnabled()) { int len = reply.length(); LOG.debug("Fetcher request verfied. enc_str=" + enc_str + ";reply=" + - reply.substring(len-len/2, len-1)); + reply.substring(len - len / 2, len - 1)); } } @@ -1554,7 +1555,7 @@ protected void sendError(ChannelHandlerContext ctx, String message, FullHttpResp } private void sendFakeShuffleHeaderWithError(ChannelHandlerContext ctx, String message, - HttpResponse response) throws IOException { + HttpResponse response) throws IOException { FullHttpResponse fullResponse = new DefaultFullHttpResponse(response.getProtocolVersion(), response.getStatus()); fullResponse.headers().set(response.headers()); @@ -1567,7 +1568,7 @@ private void sendFakeShuffleHeaderWithError(ChannelHandlerContext ctx, String me } protected void sendError(ChannelHandlerContext ctx, ByteBuf content, - FullHttpResponse response) { + FullHttpResponse response) { response.headers().set(CONTENT_TYPE, "text/plain; charset=UTF-8"); // Put shuffle version into http header response.headers().set(ShuffleHeader.HTTP_HEADER_NAME, diff --git a/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestIndexCache.java b/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestIndexCache.java index 802fdd6f60..4607647df0 100644 --- a/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestIndexCache.java +++ b/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestIndexCache.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -49,7 +49,7 @@ public class TestIndexCache { public void setUp() throws IOException { conf = new Configuration(); fs = FileSystem.getLocal(conf).getRaw(); - p = new Path(System.getProperty("test.build.data", "/tmp"), + p = new Path(System.getProperty("test.build.data", "/tmp"), "cache").makeQualified(fs.getUri(), fs.getWorkingDirectory()); } @@ -71,20 +71,20 @@ public void testLRCPolicy() throws Exception { Path f = new Path(p, Integer.toString(totalsize, 36)); writeFile(fs, f, totalsize, partsPerMap); TezIndexRecord rec = cache.getIndexInformation( - Integer.toString(totalsize, 36), r.nextInt(partsPerMap), f, - UserGroupInformation.getCurrentUser().getShortUserName()); + Integer.toString(totalsize, 36), r.nextInt(partsPerMap), f, + UserGroupInformation.getCurrentUser().getShortUserName()); checkRecord(rec, totalsize); } // delete files, ensure cache retains all elem for (FileStatus stat : fs.listStatus(p)) { - fs.delete(stat.getPath(),true); + fs.delete(stat.getPath(), true); } for (int i = bytesPerFile; i < 1024 * 1024; i += bytesPerFile) { Path f = new Path(p, Integer.toString(i, 36)); TezIndexRecord rec = cache.getIndexInformation(Integer.toString(i, 36), - r.nextInt(partsPerMap), f, - UserGroupInformation.getCurrentUser().getShortUserName()); + r.nextInt(partsPerMap), f, + UserGroupInformation.getCurrentUser().getShortUserName()); checkRecord(rec, i); } @@ -100,14 +100,13 @@ public void testLRCPolicy() throws Exception { boolean fnf = false; try { cache.getIndexInformation(Integer.toString(bytesPerFile, 36), - r.nextInt(partsPerMap), new Path(p, Integer.toString(bytesPerFile)), - UserGroupInformation.getCurrentUser().getShortUserName()); + r.nextInt(partsPerMap), new Path(p, Integer.toString(bytesPerFile)), + UserGroupInformation.getCurrentUser().getShortUserName()); } catch (IOException e) { if (e.getCause() == null || - !(e.getCause() instanceof FileNotFoundException)) { + !(e.getCause() instanceof FileNotFoundException)) { throw e; - } - else { + } else { fnf = true; } } @@ -121,8 +120,8 @@ public void testLRCPolicy() throws Exception { checkRecord(rec, i); } TezIndexRecord rec = cache.getIndexInformation(Integer.toString(totalsize, 36), - r.nextInt(partsPerMap), f, - UserGroupInformation.getCurrentUser().getShortUserName()); + r.nextInt(partsPerMap), f, + UserGroupInformation.getCurrentUser().getShortUserName()); checkRecord(rec, totalsize); } @@ -151,7 +150,7 @@ public void testBadIndex() throws Exception { dout.close(); try { cache.getIndexInformation("badindex", 7, f, - UserGroupInformation.getCurrentUser().getShortUserName()); + UserGroupInformation.getCurrentUser().getShortUserName()); fail("Did not detect bad checksum"); } catch (IOException e) { if (!(e.getCause() instanceof ChecksumException)) { @@ -178,8 +177,8 @@ public void testInvalidReduceNumberOrLength() throws Exception { try { // Number of reducers equal to partsPerMap cache.getIndexInformation("reduceEqualPartsPerMap", - partsPerMap, // reduce number == partsPerMap - feq, UserGroupInformation.getCurrentUser().getShortUserName()); + partsPerMap, // reduce number == partsPerMap + feq, UserGroupInformation.getCurrentUser().getShortUserName()); fail("Number of reducers equal to partsPerMap did not fail"); } catch (Exception e) { if (!(e instanceof IOException)) { @@ -190,9 +189,9 @@ public void testInvalidReduceNumberOrLength() throws Exception { try { // Number of reducers more than partsPerMap cache.getIndexInformation( - "reduceMorePartsPerMap", - partsPerMap + 1, // reduce number > partsPerMap - feq, UserGroupInformation.getCurrentUser().getShortUserName()); + "reduceMorePartsPerMap", + partsPerMap + 1, // reduce number > partsPerMap + feq, UserGroupInformation.getCurrentUser().getShortUserName()); fail("Number of reducers more than partsPerMap did not fail"); } catch (Exception e) { if (!(e instanceof IOException)) { @@ -218,7 +217,7 @@ public void testRemoveMap() throws Exception { final Path big = new Path(p, "bigIndex"); final String user = - UserGroupInformation.getCurrentUser().getShortUserName(); + UserGroupInformation.getCurrentUser().getShortUserName(); writeFile(fs, big, bytesPerFile, partsPerMap); // run multiple times @@ -239,7 +238,7 @@ public void run() { cache.removeMap("bigIndex"); } }; - if (i%2==0) { + if (i % 2 == 0) { getInfoThread.start(); removeMapThread.start(); } else { @@ -262,7 +261,7 @@ public void testCreateRace() throws Exception { final Path racy = new Path(p, "racyIndex"); final String user = - UserGroupInformation.getCurrentUser().getShortUserName(); + UserGroupInformation.getCurrentUser().getShortUserName(); writeFile(fs, racy, bytesPerFile, partsPerMap); // run multiple instances diff --git a/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandler.java b/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandler.java index 40b16857ad..3111c71bbf 100644 --- a/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandler.java +++ b/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandler.java @@ -20,7 +20,9 @@ //import static org.apache.hadoop.test.MetricsAsserts.assertCounter; //import static org.apache.hadoop.test.MetricsAsserts.assertGauge; //import static org.apache.hadoop.test.MetricsAsserts.getMetrics; + import org.apache.hadoop.util.DiskChecker.DiskErrorException; + import static org.junit.Assert.assertTrue; import static io.netty.buffer.Unpooled.wrappedBuffer; import static org.junit.Assert.assertEquals; @@ -109,15 +111,17 @@ public class TestShuffleHandler { private static final File TEST_DIR = new File(System.getProperty("test.build.data"), TestShuffleHandler.class.getName()).getAbsoluteFile(); private static final String HADOOP_TMP_DIR = "hadoop.tmp.dir"; + class MockShuffleHandler extends org.apache.tez.auxservices.ShuffleHandler { @Override protected Shuffle getShuffle(final Configuration conf) { return new Shuffle(conf) { @Override protected void verifyRequest(String appid, ChannelHandlerContext ctx, - HttpRequest request, HttpResponse response, URL requestUri) + HttpRequest request, HttpResponse response, URL requestUri) throws IOException { } + @Override protected MapOutputInfo getMapOutputInfo(String dagId, String mapId, Range reduceRange, String jobId, @@ -126,6 +130,7 @@ protected MapOutputInfo getMapOutputInfo(String dagId, String mapId, // Do nothing. return null; } + @Override protected void populateHeaders(List mapIds, String jobId, String dagId, String user, Range reduceRange, @@ -134,6 +139,7 @@ protected void populateHeaders(List mapIds, String jobId, Map infoMap) throws IOException { // Do nothing. } + @Override protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, Channel ch, String user, String mapId, Range reduceRange, @@ -162,9 +168,9 @@ protected Shuffle getShuffle(final Configuration conf) { return new Shuffle(conf) { @Override protected void verifyRequest(String appid, ChannelHandlerContext ctx, - HttpRequest request, HttpResponse response, URL requestUri) + HttpRequest request, HttpResponse response, URL requestUri) throws IOException { - SocketChannel channel = (SocketChannel)(ctx.channel()); + SocketChannel channel = (SocketChannel) (ctx.channel()); socketKeepAlive = channel.config().isKeepAlive(); } }; @@ -182,7 +188,7 @@ class MockShuffleHandlerWithFatalDiskError extends org.apache.tez.auxservices.Sh private JobTokenSecretManager secretManager = new JobTokenSecretManager(JobTokenSecretManager.createSecretKey(getSecret().getBytes())); - protected JobTokenSecretManager getSecretManager(){ + protected JobTokenSecretManager getSecretManager() { return secretManager; } @@ -191,26 +197,26 @@ protected Shuffle getShuffle(final Configuration conf) { return new Shuffle(conf) { @Override protected void verifyRequest(String appid, ChannelHandlerContext ctx, HttpRequest request, - HttpResponse response, URL requestUri) throws IOException { + HttpResponse response, URL requestUri) throws IOException { super.verifyRequest(appid, ctx, request, response, requestUri); } @Override protected MapOutputInfo getMapOutputInfo(String dagId, String mapId, Range reduceRange, - String jobId, String user) { + String jobId, String user) { return null; } @Override protected void populateHeaders(List mapIds, String jobId, String dagId, String user, - Range reduceRange, HttpResponse response, boolean keepAliveParam, - Map infoMap) throws IOException { + Range reduceRange, HttpResponse response, boolean keepAliveParam, + Map infoMap) throws IOException { throw new DiskErrorException(MESSAGE); } @Override protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, Channel ch, String user, - String mapId, Range reduceRange, MapOutputInfo info) throws IOException { + String mapId, Range reduceRange, MapOutputInfo info) throws IOException { return null; } }; @@ -227,8 +233,8 @@ public String getSecret() { * * @throws Exception exception */ - @Test (timeout = 10000) - public void testSerializeMeta() throws Exception { + @Test(timeout = 10000) + public void testSerializeMeta() throws Exception { assertEquals(1, ShuffleHandler.deserializeMetaData( ShuffleHandler.serializeMetaData(1))); assertEquals(-1, ShuffleHandler.deserializeMetaData( @@ -242,7 +248,7 @@ public void testSerializeMeta() throws Exception { * * @throws Exception exception */ - @Test (timeout = 10000) + @Test(timeout = 10000) public void testShuffleMetrics() throws Exception { MetricsSystem ms = new MetricsSystemImpl(); ShuffleHandler sh = new ShuffleHandler(ms); @@ -250,16 +256,16 @@ public void testShuffleMetrics() throws Exception { when(cf.isSuccess()).thenReturn(true, false); sh.metrics.shuffleConnections.incr(); - sh.metrics.shuffleOutputBytes.incr(1*MiB); + sh.metrics.shuffleOutputBytes.incr(1 * MiB); sh.metrics.shuffleConnections.incr(); - sh.metrics.shuffleOutputBytes.incr(2*MiB); + sh.metrics.shuffleOutputBytes.incr(2 * MiB); - checkShuffleMetrics(ms, 3*MiB, 0 , 0, 2); + checkShuffleMetrics(ms, 3 * MiB, 0, 0, 2); sh.metrics.operationComplete(cf); sh.metrics.operationComplete(cf); - checkShuffleMetrics(ms, 3*MiB, 1, 1, 0); + checkShuffleMetrics(ms, 3 * MiB, 1, 1, 0); sh.close(); } @@ -280,7 +286,7 @@ static void checkShuffleMetrics(MetricsSystem ms, long bytes, int failed, * * @throws Exception exception. */ - @Test (timeout = 10000) + @Test(timeout = 10000) public void testClientClosesConnection() throws Exception { final AtomicBoolean failureEncountered = new AtomicBoolean(false); Configuration conf = new Configuration(); @@ -298,6 +304,7 @@ protected MapOutputInfo getMapOutputInfo(String dagId, String mapId, throws IOException { return null; } + @Override protected void populateHeaders(List mapIds, String jobId, String dagId, String user, Range reduceRange, @@ -308,16 +315,18 @@ protected void populateHeaders(List mapIds, String jobId, // send some dummy value for content-length super.setResponseHeaders(response, keepAliveParam, 100); } + @Override protected void verifyRequest(String appid, ChannelHandlerContext ctx, - HttpRequest request, HttpResponse response, URL requestUri) - throws IOException { + HttpRequest request, HttpResponse response, URL requestUri) + throws IOException { } + @Override protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, Channel ch, String user, String mapId, Range reduceRange, MapOutputInfo info) - throws IOException { + throws IOException { // send a shuffle header and a lot of data down the channel // to trigger a broken pipe ShuffleHeader header = @@ -331,16 +340,18 @@ protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, } return ch.writeAndFlush(wrappedBuffer(dob.getData(), 0, dob.getLength())); } + @Override protected void sendError(ChannelHandlerContext ctx, - HttpResponseStatus status) { + HttpResponseStatus status) { if (failureEncountered.compareAndSet(false, true)) { ctx.channel().close(); } } + @Override protected void sendError(ChannelHandlerContext ctx, String message, - HttpResponseStatus status) { + HttpResponseStatus status) { if (failureEncountered.compareAndSet(false, true)) { ctx.channel().close(); } @@ -354,9 +365,9 @@ protected void sendError(ChannelHandlerContext ctx, String message, // simulate a reducer that closes early by reading a single shuffle header // then closing the connection URL url = new URL("http://127.0.0.1:" - + shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY) - + "/mapOutput?job=job_12345_1&dag=1&reduce=1&map=attempt_12345_1_m_1_0"); - HttpURLConnection conn = (HttpURLConnection)url.openConnection(); + + shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY) + + "/mapOutput?job=job_12345_1&dag=1&reduce=1&map=attempt_12345_1_m_1_0"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME, ShuffleHeader.DEFAULT_HTTP_HEADER_NAME); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION, @@ -376,9 +387,11 @@ protected void sendError(ChannelHandlerContext ctx, String message, static class LastSocketAddress { SocketAddress lastAddress; + void setAddress(SocketAddress lastAddress) { this.lastAddress = lastAddress; } + SocketAddress getSocketAddress() { return lastAddress; } @@ -406,9 +419,10 @@ protected MapOutputInfo getMapOutputInfo(String dagId, String mapId, throws IOException { return null; } + @Override protected void verifyRequest(String appid, ChannelHandlerContext ctx, - HttpRequest request, HttpResponse response, URL requestUri) + HttpRequest request, HttpResponse response, URL requestUri) throws IOException { } @@ -462,7 +476,7 @@ protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, @Override protected void sendError(ChannelHandlerContext ctx, - HttpResponseStatus status) { + HttpResponseStatus status) { if (failureEncountered.compareAndSet(false, true)) { ctx.channel().close(); } @@ -470,7 +484,7 @@ protected void sendError(ChannelHandlerContext ctx, @Override protected void sendError(ChannelHandlerContext ctx, String message, - HttpResponseStatus status) { + HttpResponseStatus status) { if (failureEncountered.compareAndSet(false, true)) { ctx.channel().close(); } @@ -482,27 +496,28 @@ protected void sendError(ChannelHandlerContext ctx, String message, shuffleHandler.start(); String shuffleBaseURL = "http://127.0.0.1:" - + shuffleHandler.getConfig().get( - ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY); + + shuffleHandler.getConfig().get( + ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY); URL url = new URL(shuffleBaseURL + "/mapOutput?job=job_12345_1&dag=1&reduce=1&" + "map=attempt_12345_1_m_1_0"); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME, - ShuffleHeader.DEFAULT_HTTP_HEADER_NAME); + ShuffleHeader.DEFAULT_HTTP_HEADER_NAME); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION, - ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION); + ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION); conn.connect(); DataInputStream input = new DataInputStream(conn.getInputStream()); Assert.assertEquals(HttpHeaders.Values.KEEP_ALIVE, - conn.getHeaderField(HttpHeaders.Names.CONNECTION)); + conn.getHeaderField(HttpHeaders.Names.CONNECTION)); Assert.assertEquals("timeout=1", - conn.getHeaderField(HttpHeaders.Values.KEEP_ALIVE)); + conn.getHeaderField(HttpHeaders.Values.KEEP_ALIVE)); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); ShuffleHeader header = new ShuffleHeader(); header.readFields(input); byte[] buffer = new byte[1024]; - while (input.read(buffer) != -1) {} + while (input.read(buffer) != -1) { + } SocketAddress firstAddress = lastSocketAddress.getSocketAddress(); input.close(); @@ -512,15 +527,15 @@ protected void sendError(ChannelHandlerContext ctx, String message, + "map=attempt_12345_1_m_1_0&keepAlive=true"); conn = (HttpURLConnection) url.openConnection(); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME, - ShuffleHeader.DEFAULT_HTTP_HEADER_NAME); + ShuffleHeader.DEFAULT_HTTP_HEADER_NAME); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION, - ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION); + ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION); conn.connect(); input = new DataInputStream(conn.getInputStream()); Assert.assertEquals(HttpHeaders.Values.KEEP_ALIVE, - conn.getHeaderField(HttpHeaders.Names.CONNECTION)); + conn.getHeaderField(HttpHeaders.Names.CONNECTION)); Assert.assertEquals("timeout=1", - conn.getHeaderField(HttpHeaders.Values.KEEP_ALIVE)); + conn.getHeaderField(HttpHeaders.Values.KEEP_ALIVE)); Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); header = new ShuffleHeader(); header.readFields(input); @@ -548,8 +563,8 @@ public void testSocketKeepAlive() throws Exception { shuffleHandler.start(); String shuffleBaseURL = "http://127.0.0.1:" - + shuffleHandler.getConfig().get( - ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY); + + shuffleHandler.getConfig().get( + ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY); URL url = new URL(shuffleBaseURL + "/mapOutput?job=job_12345_1&dag=1&reduce=1&" + "map=attempt_12345_1_m_1_0"); @@ -576,7 +591,7 @@ public void testSocketKeepAlive() throws Exception { * * @throws Exception exception */ - @Test (timeout = 10000) + @Test(timeout = 10000) public void testIncompatibleShuffleVersion() throws Exception { final int failureNum = 3; Configuration conf = new Configuration(); @@ -589,10 +604,10 @@ public void testIncompatibleShuffleVersion() throws Exception { // simulate a reducer that closes early by reading a single shuffle header // then closing the connection URL url = new URL("http://127.0.0.1:" - + shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY) - + "/mapOutput?job=job_12345_1&&dag=1reduce=1&map=attempt_12345_1_m_1_0"); + + shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY) + + "/mapOutput?job=job_12345_1&&dag=1reduce=1&map=attempt_12345_1_m_1_0"); for (int i = 0; i < failureNum; ++i) { - HttpURLConnection conn = (HttpURLConnection)url.openConnection(); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME, i == 0 ? "mapreduce" : "other"); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION, @@ -610,7 +625,7 @@ public void testIncompatibleShuffleVersion() throws Exception { * * @throws Exception exception */ - @Test (timeout = 10000) + @Test(timeout = 10000) public void testMaxConnections() throws Exception { Configuration conf = new Configuration(); @@ -630,6 +645,7 @@ protected MapOutputInfo getMapOutputInfo(String dagId, String mapId, // Do nothing. return null; } + @Override protected void populateHeaders(List mapIds, String jobId, String dagId, String user, Range reduceRange, @@ -638,17 +654,19 @@ protected void populateHeaders(List mapIds, String jobId, Map infoMap) throws IOException { // Do nothing. } + @Override protected void verifyRequest(String appid, ChannelHandlerContext ctx, - HttpRequest request, HttpResponse response, URL requestUri) - throws IOException { + HttpRequest request, HttpResponse response, URL requestUri) + throws IOException { // Do nothing. } + @Override protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, Channel ch, String user, String mapId, Range reduceRange, MapOutputInfo info) - throws IOException { + throws IOException { // send a shuffle header and a lot of data down the channel // to trigger a broken pipe ShuffleHeader header = @@ -657,7 +675,7 @@ protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, header.write(dob); ch.writeAndFlush(wrappedBuffer(dob.getData(), 0, dob.getLength())); dob = new DataOutputBuffer(); - for (int i=0; i<100000; ++i) { + for (int i = 0; i < 100000; ++i) { header.write(dob); } return ch.writeAndFlush(wrappedBuffer(dob.getData(), 0, dob.getLength())); @@ -674,11 +692,11 @@ protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, for (int i = 0; i < connAttempts; i++) { String URLstring = "http://127.0.0.1:" - + shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY) - + "/mapOutput?job=job_12345_1&dag=1&reduce=1&map=attempt_12345_1_m_" - + i + "_0"; + + shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY) + + "/mapOutput?job=job_12345_1&dag=1&reduce=1&map=attempt_12345_1_m_" + + i + "_0"; URL url = new URL(URLstring); - conns[i] = (HttpURLConnection)url.openConnection(); + conns[i] = (HttpURLConnection) url.openConnection(); conns[i].setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME, ShuffleHeader.DEFAULT_HTTP_HEADER_NAME); conns[i].setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION, @@ -754,7 +772,6 @@ protected void verifyRequest(String appid, ChannelHandlerContext ctx, throws IOException { // Do nothing. } - }; } }; @@ -796,8 +813,8 @@ protected void verifyRequest(String appid, ChannelHandlerContext ctx, Assert.assertEquals("Incorrect reduce id", i, header.getPartition()); headers.add(header); } - for (ShuffleHeader header: headers) { - byte[] bytes = new byte[(int)header.getCompressedLength()]; + for (ShuffleHeader header : headers) { + byte[] bytes = new byte[(int) header.getCompressedLength()]; is.read(bytes); } succeeded = true; @@ -807,7 +824,6 @@ protected void verifyRequest(String appid, ChannelHandlerContext ctx, } catch (EOFException e) { Assert.assertTrue("Failed to copy ranged fetch", succeeded); } - } finally { shuffleHandler.close(); FileUtil.fullyDelete(absLogDir); @@ -851,11 +867,10 @@ protected Shuffle getShuffle(Configuration conf) { @Override protected void verifyRequest(String appid, ChannelHandlerContext ctx, - HttpRequest request, HttpResponse response, URL requestUri) + HttpRequest request, HttpResponse response, URL requestUri) throws IOException { // Do nothing. } - }; } }; @@ -869,14 +884,14 @@ protected void verifyRequest(String appid, ChannelHandlerContext ctx, "password".getBytes(), new Text(user), new Text("shuffleService")); jt.write(outputBuffer); shuffleHandler - .initializeApplication(new ApplicationInitializationContext(user, - appId, ByteBuffer.wrap(outputBuffer.getData(), 0, - outputBuffer.getLength()))); + .initializeApplication(new ApplicationInitializationContext(user, + appId, ByteBuffer.wrap(outputBuffer.getData(), 0, + outputBuffer.getLength()))); URL url = new URL( "http://127.0.0.1:" + shuffleHandler.getConfig().get( - ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY) + ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY) + "/mapOutput?job=job_12345_0001&dag=1&reduce=" + reducerId + "&map=attempt_12345_1_m_1_0"); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); @@ -908,14 +923,14 @@ protected void verifyRequest(String appid, ChannelHandlerContext ctx, } private static void createShuffleHandlerFiles(File logDir, String user, - String appId, String appAttemptId, Configuration conf, - List fileMap) throws IOException { + String appId, String appAttemptId, Configuration conf, + List fileMap) throws IOException { String attemptDir = StringUtils.join(Path.SEPARATOR, - new String[] { logDir.getAbsolutePath(), + new String[]{logDir.getAbsolutePath(), ShuffleHandler.USERCACHE, user, - ShuffleHandler.APPCACHE, appId,"dag_1/" + "output", - appAttemptId }); + ShuffleHandler.APPCACHE, appId, "dag_1/" + "output", + appAttemptId}); File appAttemptDir = new File(attemptDir); appAttemptDir.mkdirs(); System.out.println(appAttemptDir.getAbsolutePath()); @@ -928,8 +943,8 @@ private static void createShuffleHandlerFiles(File logDir, String user, } private static void - createMapOutputFile(File mapOutputFile, Configuration conf) - throws IOException { + createMapOutputFile(File mapOutputFile, Configuration conf) + throws IOException { FileOutputStream out = new FileOutputStream(mapOutputFile); out.write("Creating new dummy map output file. Used only for testing" .getBytes()); @@ -979,7 +994,7 @@ public void testRecovery() throws IOException { jt.write(outputBuffer); shuffle.initializeApplication(new ApplicationInitializationContext(user, appId, ByteBuffer.wrap(outputBuffer.getData(), 0, - outputBuffer.getLength()))); + outputBuffer.getLength()))); // verify we are authorized to shuffle int rc = getShuffleResponseCode(shuffle, jt); @@ -1047,7 +1062,7 @@ public void testRecoveryFromOtherVersions() throws IOException { jt.write(outputBuffer); shuffle.initializeApplication(new ApplicationInitializationContext(user, appId, ByteBuffer.wrap(outputBuffer.getData(), 0, - outputBuffer.getLength()))); + outputBuffer.getLength()))); // verify we are authorized to shuffle int rc = getShuffleResponseCode(shuffle, jt); @@ -1091,15 +1106,14 @@ public void testRecoveryFromOtherVersions() throws IOException { shuffle = new ShuffleHandler(); shuffle.setRecoveryPath(new Path(tmpDir.toString())); shuffle.init(conf); - + try { shuffle.start(); Assert.fail("Incompatible version, should expect fail here."); } catch (ServiceStateException e) { Assert.assertTrue("Exception message mismatch", - e.getMessage().contains("Incompatible version for state DB schema:")); + e.getMessage().contains("Incompatible version for state DB schema:")); } - } finally { if (shuffle != null) { shuffle.close(); @@ -1109,7 +1123,7 @@ public void testRecoveryFromOtherVersions() throws IOException { } private static int getShuffleResponseCode(ShuffleHandler shuffle, - Token jt) throws IOException { + Token jt) throws IOException { URL url = new URL("http://127.0.0.1:" + shuffle.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY) + "/mapOutput?job=job_12345_0001&dag=1&reduce=0" + @@ -1165,19 +1179,22 @@ protected void populateHeaders(List mapIds, // send some dummy value for content-length super.setResponseHeaders(response, keepAliveParam, 100); } + @Override protected void verifyRequest(String appid, - ChannelHandlerContext ctx, HttpRequest request, - HttpResponse response, URL requestUri) throws IOException { + ChannelHandlerContext ctx, HttpRequest request, + HttpResponse response, URL requestUri) throws IOException { // Do nothing. } + @Override protected void sendError(ChannelHandlerContext ctx, String message, - HttpResponseStatus status) { + HttpResponseStatus status) { if (failureEncountered.compareAndSet(false, true)) { ctx.channel().close(); } } + @Override protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, Channel ch, String user, String mapId, Range reduceRange, @@ -1199,17 +1216,17 @@ protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, outputBuffer.reset(); Token jt = new Token("identifier".getBytes(), - "password".getBytes(), new Text(user), new Text("shuffleService")); + "password".getBytes(), new Text(user), new Text("shuffleService")); jt.write(outputBuffer); shuffleHandler .initializeApplication(new ApplicationInitializationContext(user, - appId, ByteBuffer.wrap(outputBuffer.getData(), 0, - outputBuffer.getLength()))); + appId, ByteBuffer.wrap(outputBuffer.getData(), 0, + outputBuffer.getLength()))); URL url = new URL( "http://127.0.0.1:" + shuffleHandler.getConfig().get( - ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY) + ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY) + "/mapOutput?job=job_12345_0001&dag=1&reduce=" + reducerId + "&map=attempt_12345_1_m_1_0"); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); @@ -1294,9 +1311,9 @@ protected void sendError(ChannelHandlerContext ctx, String message, ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION); String dagDirStr = StringUtils.join(Path.SEPARATOR, - new String[] { absLogDir.getAbsolutePath(), + new String[]{absLogDir.getAbsolutePath(), ShuffleHandler.USERCACHE, user, - ShuffleHandler.APPCACHE, appId.toString(),"dag_1/"}); + ShuffleHandler.APPCACHE, appId.toString(), "dag_1/"}); File dagDir = new File(dagDirStr); Assert.assertTrue("Dag Directory does not exist!", dagDir.exists()); conn.connect(); @@ -1322,21 +1339,21 @@ public void testVertexShuffleDelete() throws Exception { conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3); conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, - "simple"); + "simple"); UserGroupInformation.setConfiguration(conf); File absLogDir = new File("target", TestShuffleHandler.class. - getSimpleName() + "LocDir").getAbsoluteFile(); + getSimpleName() + "LocDir").getAbsoluteFile(); conf.set(YarnConfiguration.NM_LOCAL_DIRS, absLogDir.getAbsolutePath()); ApplicationId appId = ApplicationId.newInstance(12345L, 1); String appAttemptId = "attempt_12345_0001_1_00_000000_0_10003_0"; String user = "randomUser"; List fileMap = new ArrayList(); - String vertexDirStr = StringUtils.join(Path.SEPARATOR, new String[] { absLogDir.getAbsolutePath(), + String vertexDirStr = StringUtils.join(Path.SEPARATOR, new String[]{absLogDir.getAbsolutePath(), ShuffleHandler.USERCACHE, user, ShuffleHandler.APPCACHE, appId.toString(), "dag_1/output/" + appAttemptId}); File vertexDir = new File(vertexDirStr); Assert.assertFalse("vertex directory should not be present", vertexDir.exists()); createShuffleHandlerFiles(absLogDir, user, appId.toString(), appAttemptId, - conf, fileMap); + conf, fileMap); ShuffleHandler shuffleHandler = new ShuffleHandler() { @Override protected Shuffle getShuffle(Configuration conf) { @@ -1359,24 +1376,24 @@ protected void sendError(ChannelHandlerContext ctx, String message, DataOutputBuffer outputBuffer = new DataOutputBuffer(); outputBuffer.reset(); Token jt = - new Token("identifier".getBytes(), - "password".getBytes(), new Text(user), new Text("shuffleService")); + new Token("identifier".getBytes(), + "password".getBytes(), new Text(user), new Text("shuffleService")); jt.write(outputBuffer); shuffleHandler - .initializeApplication(new ApplicationInitializationContext(user, - appId, ByteBuffer.wrap(outputBuffer.getData(), 0, - outputBuffer.getLength()))); + .initializeApplication(new ApplicationInitializationContext(user, + appId, ByteBuffer.wrap(outputBuffer.getData(), 0, + outputBuffer.getLength()))); URL url = - new URL( - "http://127.0.0.1:" - + shuffleHandler.getConfig().get( - ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY) - + "/mapOutput?vertexAction=delete&job=job_12345_0001&dag=1&vertex=00"); + new URL( + "http://127.0.0.1:" + + shuffleHandler.getConfig().get( + ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY) + + "/mapOutput?vertexAction=delete&job=job_12345_0001&dag=1&vertex=00"); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_NAME, - ShuffleHeader.DEFAULT_HTTP_HEADER_NAME); + ShuffleHeader.DEFAULT_HTTP_HEADER_NAME); conn.setRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION, - ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION); + ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION); Assert.assertTrue("Attempt Directory does not exist!", vertexDir.exists()); conn.connect(); try { @@ -1409,10 +1426,10 @@ public void testFailedTaskAttemptDelete() throws Exception { String user = "randomUser"; List fileMap = new ArrayList(); String taskAttemptDirStr = - StringUtils.join(Path.SEPARATOR, - new String[] {absLogDir.getAbsolutePath(), - ShuffleHandler.USERCACHE, user, - ShuffleHandler.APPCACHE, appId.toString(), "dag_1/output/", appAttemptId}); + StringUtils.join(Path.SEPARATOR, + new String[]{absLogDir.getAbsolutePath(), + ShuffleHandler.USERCACHE, user, + ShuffleHandler.APPCACHE, appId.toString(), "dag_1/output/", appAttemptId}); File taskAttemptDir = new File(taskAttemptDirStr); Assert.assertFalse("Task Attempt Directory should not exist", taskAttemptDir.exists()); createShuffleHandlerFiles(absLogDir, user, appId.toString(), appAttemptId, @@ -1506,13 +1523,13 @@ public void testSendMapCount() throws Exception { conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0); sh.init(conf); sh.start(); - int maxOpenFiles =conf.getInt(ShuffleHandler.SHUFFLE_MAX_SESSION_OPEN_FILES, + int maxOpenFiles = conf.getInt(ShuffleHandler.SHUFFLE_MAX_SESSION_OPEN_FILES, ShuffleHandler.DEFAULT_SHUFFLE_MAX_SESSION_OPEN_FILES); sh.getShuffle(conf).channelRead(mockCtx, httpRequest); assertTrue("Number of Open files should not exceed the configured " + "value!-Not Expected", listenerList.size() <= maxOpenFiles); - while(!listenerList.isEmpty()) { + while (!listenerList.isEmpty()) { listenerList.remove(0).operationComplete(mockFuture); assertTrue("Number of Open files should not exceed the configured " + "value!-Not Expected", @@ -1569,7 +1586,7 @@ public void testShuffleHandlerSendsDiskError() throws Exception { } public ChannelFuture createMockChannelFuture(Channel mockCh, - final List listenerList) { + final List listenerList) { final ChannelFuture mockFuture = mock(ChannelFuture.class); when(mockFuture.channel()).thenReturn(mockCh); doReturn(true).when(mockFuture).isSuccess(); diff --git a/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandlerJobs.java b/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandlerJobs.java index 380163b526..0893441e6b 100644 --- a/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandlerJobs.java +++ b/tez-plugins/tez-aux-services/src/test/java/org/apache/tez/auxservices/TestShuffleHandlerJobs.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,6 +17,7 @@ */ package org.apache.tez.auxservices; + import java.io.File; import java.io.IOException; @@ -30,8 +31,10 @@ import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.examples.OrderedWordCount; import org.apache.tez.runtime.library.api.TezRuntimeConfiguration; + import static org.apache.tez.test.TestTezJobs.generateOrderedWordCountInput; import static org.apache.tez.test.TestTezJobs.verifyOutput; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -56,6 +59,7 @@ public class TestShuffleHandlerJobs { private static FileSystem remoteFs; private static int NUM_NMS = 5; private static int NUM_DNS = 5; + @BeforeClass public static void setup() throws IOException { try { @@ -89,7 +93,6 @@ public static void setup() throws IOException { tezCluster.init(conf); tezCluster.start(); } - } @AfterClass @@ -103,6 +106,7 @@ public static void tearDown() { dfsCluster = null; } } + @Test(timeout = 300000) public void testOrderedWordCount() throws Exception { String inputDirStr = "/tmp/owc-input/"; @@ -127,12 +131,12 @@ public void testOrderedWordCount() throws Exception { try { final OrderedWordCount job = new OrderedWordCount(); Assert.assertTrue("OrderedWordCount failed", job.run(tezConf, new String[]{"-counter", - inputDirStr, outputDirStr, "10"}, tezSession)==0); + inputDirStr, outputDirStr, "10"}, tezSession) == 0); verifyOutput(outputDir, remoteFs); tezSession.stop(); ClientRMService rmService = tezCluster.getResourceManager().getClientRMService(); boolean isAppComplete = false; - while(!isAppComplete) { + while (!isAppComplete) { GetApplicationReportResponse resp = rmService.getApplicationReport( new GetApplicationReportRequest() { @Override @@ -149,7 +153,7 @@ public void setApplicationId(ApplicationId applicationId) { } Thread.sleep(100); } - for(int i = 0; i < NUM_NMS; i++) { + for (int i = 0; i < NUM_NMS; i++) { String appPath = tezCluster.getTestWorkDir() + "/" + this.getClass().getName() + "-localDir-nm-" + i + "_0/usercache/" + UserGroupInformation.getCurrentUser().getUserName() + "/appcache/" + job.getAppId(); diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/ATSImportTool.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/ATSImportTool.java index c909f7aa0d..fc6047fc22 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/ATSImportTool.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/ATSImportTool.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -362,12 +362,12 @@ private static Options buildOptions() { .withDescription("DagId that needs to be downloaded").hasArg().isRequired(true).create(); Option downloadDirOption = OptionBuilder.withArgName(BASE_DOWNLOAD_DIR).withLongOpt - (BASE_DOWNLOAD_DIR) + (BASE_DOWNLOAD_DIR) .withDescription("Download directory where data needs to be downloaded").hasArg() .isRequired(true).create(); Option atsAddressOption = OptionBuilder.withArgName(YARN_TIMELINE_SERVICE_ADDRESS).withLongOpt( - YARN_TIMELINE_SERVICE_ADDRESS) + YARN_TIMELINE_SERVICE_ADDRESS) .withDescription("Optional. ATS address (e.g http://clusterATSNode:8188)").hasArg() .isRequired(false) .create(); diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ATSData.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ATSData.java index f504007e8f..973a933ea8 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ATSData.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ATSData.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -44,5 +44,4 @@ public interface ATSData { * @throws TezException */ public DagInfo getDAGData(String dagId) throws TezException; - } diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ATSFileParser.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ATSFileParser.java index caeb406f21..641b61480b 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ATSFileParser.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/ATSFileParser.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -162,7 +162,7 @@ private void processApplication(JSONObject tezApplicationJson) throws JSONExcept this.config = Maps.newHashMap(); JSONObject configNode = otherInfoNode.getJSONObject(Constants.CONFIG); Iterator it = configNode.keys(); - while(it.hasNext()) { + while (it.hasNext()) { String key = (String) it.next(); String value = configNode.getString(key); config.put(key, value); diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java index b57f4f0e65..e868f0ec78 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java @@ -69,9 +69,13 @@ public SimpleHistoryParser(List files) { protected interface JSONObjectSource { boolean hasNext() throws IOException; + JSONObject next() throws JSONException; + void close(); - }; + } + + ; /** * Get in-memory representation of DagInfo @@ -105,7 +109,7 @@ private void populateOtherInfo(JSONObject source, JSONObject destination) throws } private void populateOtherInfo(JSONObject source, String entityName, - Map destMap) throws JSONException { + Map destMap) throws JSONException { JSONObject destinationJson = destMap.get(entityName); JSONObject destOtherInfo = destinationJson.getJSONObject(Constants.OTHER_INFO); populateOtherInfo(source, destOtherInfo); @@ -153,7 +157,7 @@ protected void parse(String dagId, JSONObjectSource source) } protected void postProcessMaps(Map vertexJsonMap, - Map taskJsonMap, Map attemptJsonMap) + Map taskJsonMap, Map attemptJsonMap) throws JSONException { for (JSONObject jsonObject : vertexJsonMap.values()) { VertexInfo vertexInfo = VertexInfo.create(jsonObject); @@ -214,8 +218,8 @@ protected void postProcessMaps(Map vertexJsonMap, } protected void readEventsFromSource(String dagId, JSONObjectSource source, - Map vertexJsonMap, Map taskJsonMap, - Map attemptJsonMap) throws JSONException, TezException, IOException{ + Map vertexJsonMap, Map taskJsonMap, + Map attemptJsonMap) throws JSONException, TezException, IOException { JSONObject dagJson = null; TezDAGID tezDAGID = TezDAGID.fromString(dagId); String userName = null; @@ -226,87 +230,87 @@ protected void readEventsFromSource(String dagId, JSONObjectSource source, String entity = jsonObject.getString(Constants.ENTITY); String entityType = jsonObject.getString(Constants.ENTITY_TYPE); switch (entityType) { - case Constants.TEZ_DAG_ID: - if (!dagId.equals(entity)) { - LOG.warn(dagId + " is not matching with " + entity); - continue; - } - // Club all DAG related information together (DAG_INIT, DAG_FINISH etc). Each of them - // would have a set of entities in otherinfo (e.g vertex mapping, dagPlan, start/finish - // time etc). - if (dagJson == null) { - dagJson = jsonObject; - } else { - if (dagJson.optJSONObject(ATSConstants.OTHER_INFO).optJSONObject(ATSConstants.DAG_PLAN) == null) { - // if DAG_PLAN is not filled already, let's try to fetch it from other - dagJson.getJSONObject(ATSConstants.OTHER_INFO).put(ATSConstants.DAG_PLAN, - jsonObject.getJSONObject(ATSConstants.OTHER_INFO).getJSONObject(ATSConstants.DAG_PLAN)); + case Constants.TEZ_DAG_ID: + if (!dagId.equals(entity)) { + LOG.warn(dagId + " is not matching with " + entity); + continue; } - mergeSubJSONArray(jsonObject, dagJson, Constants.EVENTS); - } - JSONArray relatedEntities = dagJson.optJSONArray(Constants - .RELATED_ENTITIES); - //UserName is present in related entities - // {"entity":"userXYZ","entitytype":"user"} - if (relatedEntities != null) { - for (int i = 0; i < relatedEntities.length(); i++) { - JSONObject subEntity = relatedEntities.getJSONObject(i); - String subEntityType = subEntity.optString(Constants.ENTITY_TYPE); - if (subEntityType != null && subEntityType.equals(Constants.USER)) { - userName = subEntity.getString(Constants.ENTITY); - break; + // Club all DAG related information together (DAG_INIT, DAG_FINISH etc). Each of them + // would have a set of entities in otherinfo (e.g vertex mapping, dagPlan, start/finish + // time etc). + if (dagJson == null) { + dagJson = jsonObject; + } else { + if (dagJson.optJSONObject(ATSConstants.OTHER_INFO).optJSONObject(ATSConstants.DAG_PLAN) == null) { + // if DAG_PLAN is not filled already, let's try to fetch it from other + dagJson.getJSONObject(ATSConstants.OTHER_INFO).put(ATSConstants.DAG_PLAN, + jsonObject.getJSONObject(ATSConstants.OTHER_INFO).getJSONObject(ATSConstants.DAG_PLAN)); } + mergeSubJSONArray(jsonObject, dagJson, Constants.EVENTS); } - } - populateOtherInfo(jsonObject.optJSONObject(Constants.OTHER_INFO), - dagJson.getJSONObject(Constants.OTHER_INFO)); - break; - case Constants.TEZ_VERTEX_ID: - String vertexName = entity; - TezVertexID tezVertexID = TezVertexID.fromString(vertexName); - if (!tezDAGID.equals(tezVertexID.getDAGID())) { - LOG.warn("{} does not belong to {} ('{}' != '{}')}", vertexName, tezDAGID, tezDAGID, tezVertexID.getDAGID()); - continue; - } - if (!vertexJsonMap.containsKey(vertexName)) { - vertexJsonMap.put(vertexName, jsonObject); - } else { - mergeSubJSONArray(jsonObject, vertexJsonMap.get(vertexName), Constants.EVENTS); - } - populateOtherInfo(jsonObject.optJSONObject(Constants.OTHER_INFO), vertexName, vertexJsonMap); - break; - case Constants.TEZ_TASK_ID: - String taskName = entity; - TezTaskID tezTaskID = TezTaskID.fromString(taskName); - if (!tezDAGID.equals(tezTaskID.getDAGID())) { - LOG.warn("{} does not belong to {} ('{}' != '{}')}", taskName, tezDAGID, tezDAGID, - tezTaskID.getDAGID()); - continue; - } - if (!taskJsonMap.containsKey(taskName)) { - taskJsonMap.put(taskName, jsonObject); - } else { - mergeSubJSONArray(jsonObject, taskJsonMap.get(taskName), Constants.EVENTS); - } - populateOtherInfo(jsonObject.optJSONObject(Constants.OTHER_INFO), taskName, taskJsonMap); - break; - case Constants.TEZ_TASK_ATTEMPT_ID: - String taskAttemptName = entity; - TezTaskAttemptID tezAttemptId = TezTaskAttemptID.fromString(taskAttemptName); - if (!tezDAGID.equals(tezAttemptId.getDAGID())) { - LOG.warn("{} does not belong to {} ('{}' != '{}')}", taskAttemptName, tezDAGID, tezDAGID, - tezAttemptId.getDAGID()); - continue; - } - if (!attemptJsonMap.containsKey(taskAttemptName)) { - attemptJsonMap.put(taskAttemptName, jsonObject); - } else { - mergeSubJSONArray(jsonObject, attemptJsonMap.get(taskAttemptName), Constants.EVENTS); - } - populateOtherInfo(jsonObject.optJSONObject(Constants.OTHER_INFO), taskAttemptName, attemptJsonMap); - break; - default: - break; + JSONArray relatedEntities = dagJson.optJSONArray(Constants + .RELATED_ENTITIES); + //UserName is present in related entities + // {"entity":"userXYZ","entitytype":"user"} + if (relatedEntities != null) { + for (int i = 0; i < relatedEntities.length(); i++) { + JSONObject subEntity = relatedEntities.getJSONObject(i); + String subEntityType = subEntity.optString(Constants.ENTITY_TYPE); + if (subEntityType != null && subEntityType.equals(Constants.USER)) { + userName = subEntity.getString(Constants.ENTITY); + break; + } + } + } + populateOtherInfo(jsonObject.optJSONObject(Constants.OTHER_INFO), + dagJson.getJSONObject(Constants.OTHER_INFO)); + break; + case Constants.TEZ_VERTEX_ID: + String vertexName = entity; + TezVertexID tezVertexID = TezVertexID.fromString(vertexName); + if (!tezDAGID.equals(tezVertexID.getDAGID())) { + LOG.warn("{} does not belong to {} ('{}' != '{}')}", vertexName, tezDAGID, tezDAGID, tezVertexID.getDAGID()); + continue; + } + if (!vertexJsonMap.containsKey(vertexName)) { + vertexJsonMap.put(vertexName, jsonObject); + } else { + mergeSubJSONArray(jsonObject, vertexJsonMap.get(vertexName), Constants.EVENTS); + } + populateOtherInfo(jsonObject.optJSONObject(Constants.OTHER_INFO), vertexName, vertexJsonMap); + break; + case Constants.TEZ_TASK_ID: + String taskName = entity; + TezTaskID tezTaskID = TezTaskID.fromString(taskName); + if (!tezDAGID.equals(tezTaskID.getDAGID())) { + LOG.warn("{} does not belong to {} ('{}' != '{}')}", taskName, tezDAGID, tezDAGID, + tezTaskID.getDAGID()); + continue; + } + if (!taskJsonMap.containsKey(taskName)) { + taskJsonMap.put(taskName, jsonObject); + } else { + mergeSubJSONArray(jsonObject, taskJsonMap.get(taskName), Constants.EVENTS); + } + populateOtherInfo(jsonObject.optJSONObject(Constants.OTHER_INFO), taskName, taskJsonMap); + break; + case Constants.TEZ_TASK_ATTEMPT_ID: + String taskAttemptName = entity; + TezTaskAttemptID tezAttemptId = TezTaskAttemptID.fromString(taskAttemptName); + if (!tezDAGID.equals(tezAttemptId.getDAGID())) { + LOG.warn("{} does not belong to {} ('{}' != '{}')}", taskAttemptName, tezDAGID, tezDAGID, + tezAttemptId.getDAGID()); + continue; + } + if (!attemptJsonMap.containsKey(taskAttemptName)) { + attemptJsonMap.put(taskAttemptName, jsonObject); + } else { + mergeSubJSONArray(jsonObject, attemptJsonMap.get(taskAttemptName), Constants.EVENTS); + } + populateOtherInfo(jsonObject.optJSONObject(Constants.OTHER_INFO), taskAttemptName, attemptJsonMap); + break; + default: + break; } } source.close(); @@ -332,4 +336,4 @@ private void mergeSubJSONArray(JSONObject source, JSONObject destination, String destination.getJSONArray(key).put(source.getJSONArray(key).get(i)); } } -} \ No newline at end of file +} diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/AdditionalInputOutputDetails.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/AdditionalInputOutputDetails.java index 7e04b8c05b..70d7ae8f82 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/AdditionalInputOutputDetails.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/AdditionalInputOutputDetails.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,7 +34,7 @@ public class AdditionalInputOutputDetails { private final String userPayloadText; public AdditionalInputOutputDetails(String name, String clazz, String initializer, - String userPayloadText) { + String userPayloadText) { this.name = name; this.clazz = clazz; this.initializer = initializer; diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseInfo.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseInfo.java index 783f486a15..c92a464a22 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseInfo.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseInfo.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -136,7 +136,6 @@ public Map getCounter(String counterGroupName, String counte //Retrieved counter specific to a counter group. Safe to exit. break; } - } return result; } @@ -150,5 +149,4 @@ public Map getCounter(String counterGroupName, String counte public Map getCounter(String counter) { return getCounter(null, counter); } - } diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java index 9f3881c8b2..f2f10407b4 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/BaseParser.java @@ -38,14 +38,12 @@ public abstract class BaseParser { protected final List taskList; protected final List attemptList; - public BaseParser() { vertexList = Lists.newLinkedList(); taskList = Lists.newLinkedList(); attemptList = Lists.newLinkedList(); } - protected boolean checkFiles(List files) { if (files.isEmpty()) { return false; @@ -58,7 +56,6 @@ protected boolean checkFiles(List files) { return true; } - protected void addRawDataToDagInfo() { dagInfo.addMeta("vertices", vertexList); dagInfo.addMeta("tasks", taskList); @@ -104,7 +101,6 @@ protected void linkParsedContents() { } } - //Set reference time for all events for (VertexInfo vertexInfo : dagInfo.getVertices()) { setReferenceTime(vertexInfo.getEvents(), dagInfo.getStartTimeInterval()); @@ -128,7 +124,8 @@ protected void linkParsedContents() { */ private void setReferenceTime(List eventList, final long referenceTime) { Iterables.all(eventList, new Predicate() { - @Override public boolean apply(Event input) { + @Override + public boolean apply(Event input) { input.setReferenceTime(referenceTime); return false; } diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/Constants.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/Constants.java index dce79e2f78..7d717a4fa7 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/Constants.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/Constants.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -58,7 +58,4 @@ public class Constants extends ATSConstants { public static final String TASKS = "tasks"; public static final String TASK_ATTEMPTS = "task_attempts"; public static final String APPLICATION = "application"; - - - } diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/Container.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/Container.java index 4e01d1b4be..52bae44665 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/Container.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/Container.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -43,7 +43,8 @@ public final String getHost() { return host; } - @Override public String toString() { + @Override + public String toString() { StringBuilder sb = new StringBuilder(); sb.append("["); sb.append("id=").append(id).append(", "); @@ -52,11 +53,13 @@ public final String getHost() { return sb.toString(); } - @Override public int hashCode() { + @Override + public int hashCode() { return Objects.hashCode(id, host); } - @Override public boolean equals(Object obj) { + @Override + public boolean equals(Object obj) { if (this == obj) { return true; } diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java index 5067ec7aa8..26c6baff7c 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -106,7 +106,7 @@ public class DagInfo extends BaseInfo { JSONObject otherInfoNode = jsonObject.getJSONObject(Constants.OTHER_INFO); long sTime = otherInfoNode.optLong(Constants.START_TIME); - long eTime= otherInfoNode.optLong(Constants.FINISH_TIME); + long eTime = otherInfoNode.optLong(Constants.FINISH_TIME); userName = otherInfoNode.optString(Constants.USER); if (eTime < sTime) { LOG.warn("DAG has got wrong start/end values. " @@ -114,16 +114,16 @@ public class DagInfo extends BaseInfo { + "timestamps in DAG started/finished events"); // Check if events DAG_STARTED, DAG_FINISHED can be made use of - for(Event event : eventList) { + for (Event event : eventList) { switch (HistoryEventType.valueOf(event.getType())) { - case DAG_STARTED: - sTime = event.getAbsoluteTime(); - break; - case DAG_FINISHED: - eTime = event.getAbsoluteTime(); - break; - default: - break; + case DAG_STARTED: + sTime = event.getAbsoluteTime(); + break; + case DAG_FINISHED: + eTime = event.getAbsoluteTime(); + break; + default: + break; } } @@ -203,7 +203,6 @@ private void parseDAGContext(JSONObject callerContextInfo) { } else { LOG.info("No DAG Caller Context Id and Type available"); } - } private void parseBasicVertexInfo(JSONArray verticesInfo) throws JSONException { @@ -273,7 +272,6 @@ private AdditionalInputOutputDetails[] parseAdditionalDetailsForVertex(JSONArray additionalInputOutputDetails[j] = new AdditionalInputOutputDetails(name, clazz, initializer, userPayloadText); - } return additionalInputOutputDetails; } @@ -502,7 +500,8 @@ public final List getVertices() { List vertices = Lists.newLinkedList(vertexNameMap.values()); Collections.sort(vertices, new Comparator() { - @Override public int compare(VertexInfo o1, VertexInfo o2) { + @Override + public int compare(VertexInfo o1, VertexInfo o2) { return (o1.getStartTimeInterval() < o2.getStartTimeInterval()) ? -1 : ((o1.getStartTimeInterval() == o2.getStartTimeInterval()) ? 0 : 1); @@ -547,7 +546,8 @@ public final List getSuccessfullVertices() { public final List getVertices(final VertexState state) { return Collections.unmodifiableList(Lists.newLinkedList(Iterables.filter(Lists.newLinkedList (vertexNameMap.values()), new Predicate() { - @Override public boolean apply(VertexInfo input) { + @Override + public boolean apply(VertexInfo input) { return input.getStatus() != null && input.getStatus().equals(state.toString()); } } @@ -562,7 +562,8 @@ public final Map getVertexMapping() { private Ordering getVertexOrdering() { return Ordering.from(new Comparator() { - @Override public int compare(VertexInfo o1, VertexInfo o2) { + @Override + public int compare(VertexInfo o1, VertexInfo o2) { return (o1.getTimeTaken() < o2.getTimeTaken()) ? -1 : ((o1.getTimeTaken() == o2.getTimeTaken()) ? 0 : 1); diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/EdgeInfo.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/EdgeInfo.java index ab8e831874..1d990f3015 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/EdgeInfo.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/EdgeInfo.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -37,8 +37,8 @@ public class EdgeInfo { private VertexInfo destinationVertex; public EdgeInfo(String inputVertexName, String outputVertexName, String dataMovementType, - String edgeSourceClass, String edgeDestinationClass, String inputUserPayloadAsText, String - outputUserPayloadAsText) { + String edgeSourceClass, String edgeDestinationClass, String inputUserPayloadAsText, String + outputUserPayloadAsText) { this.inputVertexName = inputVertexName; this.outputVertexName = outputVertexName; this.dataMovementType = dataMovementType; diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/Event.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/Event.java index 70310f3201..e7b41744c8 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/Event.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/Event.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskAttemptInfo.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskAttemptInfo.java index c45fcf5ff3..37c5ad81a4 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskAttemptInfo.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskAttemptInfo.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -74,17 +74,20 @@ public class TaskAttemptInfo extends BaseInfo { private TaskInfo taskInfo; private Container container; - + public static class DataDependencyEvent { String taId; long timestamp; + public DataDependencyEvent(String id, long time) { taId = id; timestamp = time; } + public long getTimestamp() { return timestamp; } + public String getTaskAttemptId() { return taId; } @@ -110,16 +113,16 @@ public String getTaskAttemptId() { + "timestamps in DAG started/finished events"); // Check if events TASK_STARTED, TASK_FINISHED can be made use of - for(Event event : eventList) { + for (Event event : eventList) { switch (HistoryEventType.valueOf(event.getType())) { - case TASK_ATTEMPT_STARTED: - sTime = event.getAbsoluteTime(); - break; - case TASK_ATTEMPT_FINISHED: - eTime = event.getAbsoluteTime(); - break; - default: - break; + case TASK_ATTEMPT_STARTED: + sTime = event.getAbsoluteTime(); + break; + case TASK_ATTEMPT_FINISHED: + eTime = event.getAbsoluteTime(); + break; + default: + break; } } @@ -158,7 +161,7 @@ public String getTaskAttemptId() { .intern(otherInfoNode.optString(ATSConstants.TASK_ATTEMPT_ERROR_ENUM)); executionTimeInterval = (endTime > startTime) ? (endTime - startTime) : 0; } - + public static Ordering orderingOnAllocationTime() { return Ordering.from(new Comparator() { @Override @@ -183,26 +186,26 @@ public final long getStartTimeInterval() { public final long getFinishTimeInterval() { return endTime - (getTaskInfo().getVertexInfo().getDagInfo().getStartTime()); } - + public final boolean isSucceeded() { return status.equals(SUCCEEDED); } - + public final List getLastDataEvents() { return lastDataEvents; } - + public final long getExecutionTimeInterval() { return executionTimeInterval; } - + public final long getPostDataExecutionTimeInterval() { if (getStartTime() > 0 && getFinishTime() > 0) { // start time defaults to the actual start time long postDataStartTime = startTime; if (getLastDataEvents() != null && !getLastDataEvents().isEmpty()) { // if last data event is after the start time then use last data event time - long lastEventTime = getLastDataEvents().get(getLastDataEvents().size()-1).getTimestamp(); + long lastEventTime = getLastDataEvents().get(getLastDataEvents().size() - 1).getTimestamp(); postDataStartTime = startTime > lastEventTime ? startTime : lastEventTime; } return (getFinishTime() - postDataStartTime); @@ -213,11 +216,11 @@ public final long getPostDataExecutionTimeInterval() { public final long getAllocationToEndTimeInterval() { return (endTime - allocationTime); } - + public final long getAllocationToStartTimeInterval() { return (startTime - allocationTime); } - + public final long getCreationToAllocationTimeInterval() { return (allocationTime - creationTime); } @@ -233,9 +236,9 @@ public final long getFinishTime() { public final long getCreationTime() { return creationTime; } - + public final DataDependencyEvent getLastDataEventInfo(long timeThreshold) { - for (int i=lastDataEvents.size()-1; i>=0; i--) { + for (int i = lastDataEvents.size() - 1; i >= 0; i--) { // walk back in time until we get first event that happened before the threshold DataDependencyEvent item = lastDataEvents.get(i); if (item.getTimestamp() < timeThreshold) { @@ -244,7 +247,7 @@ public final DataDependencyEvent getLastDataEventInfo(long timeThreshold) { } return null; } - + public final long getTimeTaken() { return getFinishTimeInterval() - getStartTimeInterval(); } @@ -252,7 +255,7 @@ public final long getTimeTaken() { public final long getCreationTimeInterval() { return creationTime - (getTaskInfo().getVertexInfo().getDagInfo().getStartTime()); } - + public final String getCreationCausalTA() { return creationCausalTA; } @@ -260,17 +263,17 @@ public final String getCreationCausalTA() { public final long getAllocationTime() { return allocationTime; } - + public final String getShortName() { - return getTaskInfo().getVertexInfo().getVertexName() + " : " + - taskAttemptId.substring(taskAttemptId.lastIndexOf('_', taskAttemptId.lastIndexOf('_') - 1) + 1); + return getTaskInfo().getVertexInfo().getVertexName() + " : " + + taskAttemptId.substring(taskAttemptId.lastIndexOf('_', taskAttemptId.lastIndexOf('_') - 1) + 1); } @Override public final String getDiagnostics() { return diagnostics; } - + public final String getTerminationCause() { return terminationCause; } @@ -293,7 +296,7 @@ public final boolean isLocalityInfoAvailable() { } return false; } - + public final String getDetailedStatus() { if (!Strings.isNullOrEmpty(getTerminationCause())) { return getStatus() + ":" + getTerminationCause(); diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskInfo.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskInfo.java index fd01093244..285175f538 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskInfo.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskInfo.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -85,16 +85,16 @@ public class TaskInfo extends BaseInfo { + "timestamps in DAG started/finished events"); // Check if events TASK_STARTED, TASK_FINISHED can be made use of - for(Event event : eventList) { + for (Event event : eventList) { switch (HistoryEventType.valueOf(event.getType())) { - case TASK_STARTED: - sTime = event.getAbsoluteTime(); - break; - case TASK_FINISHED: - eTime = event.getAbsoluteTime(); - break; - default: - break; + case TASK_STARTED: + sTime = event.getAbsoluteTime(); + break; + case TASK_FINISHED: + eTime = event.getAbsoluteTime(); + break; + default: + break; } } @@ -128,7 +128,7 @@ public final long getFinishTime() { @Override public final long getFinishTimeInterval() { - long taskFinishTime = endTime - (vertexInfo.getDagInfo().getStartTime()); + long taskFinishTime = endTime - (vertexInfo.getDagInfo().getStartTime()); if (taskFinishTime < 0) { //probably vertex is not complete or failed in middle. get the last task attempt time for (TaskAttemptInfo attemptInfo : getTaskAttempts()) { @@ -267,7 +267,8 @@ public final TaskAttemptInfo getLastTaskAttemptToFinish() { } return Ordering.from(new Comparator() { - @Override public int compare(TaskAttemptInfo o1, TaskAttemptInfo o2) { + @Override + public int compare(TaskAttemptInfo o1, TaskAttemptInfo o2) { return (o1.getFinishTimeInterval() < o2.getFinishTimeInterval()) ? -1 : ((o1.getFinishTimeInterval() == o2.getFinishTimeInterval()) ? 0 : 1); @@ -294,7 +295,8 @@ public final float getAvgTaskAttemptDuration() { private Ordering orderingOnTimeTaken() { return Ordering.from(new Comparator() { - @Override public int compare(TaskAttemptInfo o1, TaskAttemptInfo o2) { + @Override + public int compare(TaskAttemptInfo o1, TaskAttemptInfo o2) { return (o1.getTimeTaken() < o2.getTimeTaken()) ? -1 : ((o1.getTimeTaken() == o2.getTimeTaken()) ? 0 : 1); @@ -304,7 +306,8 @@ private Ordering orderingOnTimeTaken() { private Ordering orderingOnAttemptStartTime() { return Ordering.from(new Comparator() { - @Override public int compare(TaskAttemptInfo o1, TaskAttemptInfo o2) { + @Override + public int compare(TaskAttemptInfo o1, TaskAttemptInfo o2) { return (o1.getStartTimeInterval() < o2.getStartTimeInterval()) ? -1 : ((o1.getStartTimeInterval() == o2.getStartTimeInterval()) ? 0 : 1); } diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VersionInfo.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VersionInfo.java index 861e50f97b..68e826e8e0 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VersionInfo.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VersionInfo.java @@ -41,5 +41,4 @@ public String getRevision() { public String getVersion() { return version; } - } diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VertexInfo.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VertexInfo.java index 106f2bc729..7a7f4400a8 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VertexInfo.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VertexInfo.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -59,7 +59,7 @@ public class VertexInfo extends BaseInfo { private final long initRequestedTime; private final long startTime; private final long startRequestedTime; - + private final String diagnostics; private final String processorClass; @@ -80,7 +80,7 @@ public class VertexInfo extends BaseInfo { private final List additionalInputInfoList; private final List additionalOutputInfoList; - + private long avgPostDataExecutionTimeInterval = -1; private DagInfo dagInfo; @@ -114,19 +114,19 @@ public class VertexInfo extends BaseInfo { + "timestamps in DAG started/finished events"); // Check if events VERTEX_STARTED, VERTEX_FINISHED can be made use of - for(Event event : eventList) { + for (Event event : eventList) { switch (HistoryEventType.valueOf(event.getType())) { - case VERTEX_INITIALIZED: - iTime = event.getAbsoluteTime(); - break; - case VERTEX_STARTED: - sTime = event.getAbsoluteTime(); - break; - case VERTEX_FINISHED: - eTime = event.getAbsoluteTime(); - break; - default: - break; + case VERTEX_INITIALIZED: + iTime = event.getAbsoluteTime(); + break; + case VERTEX_STARTED: + sTime = event.getAbsoluteTime(); + break; + case VERTEX_FINISHED: + eTime = event.getAbsoluteTime(); + break; + default: + break; } } @@ -139,7 +139,6 @@ public class VertexInfo extends BaseInfo { finishTime = eTime; initTime = iTime; - diagnostics = otherInfoNode.optString(Constants.DIAGNOSTICS); numTasks = otherInfoNode.optInt(Constants.NUM_TASKS); failedTasks = otherInfoNode.optInt(Constants.NUM_FAILED_TASKS); @@ -184,7 +183,7 @@ void setAdditionalInputInfoList(List additionalInp this.additionalInputInfoList.clear(); this.additionalInputInfoList.addAll(additionalInputInfoList); } - + void setAdditionalOutputInfoList(List additionalOutputInfoList) { this.additionalOutputInfoList.clear(); this.additionalOutputInfoList.addAll(additionalOutputInfoList); @@ -228,11 +227,11 @@ public final long getFirstTaskStartTimeInterval() { public final long getLastTaskFinishTimeInterval() { if (getLastTaskToFinish() == null || getLastTaskToFinish().getFinishTimeInterval() < 0) { - return dagInfo.getFinishTimeInterval(); + return dagInfo.getFinishTimeInterval(); } return getLastTaskToFinish().getFinishTimeInterval(); } - + public final long getAvgPostDataExecutionTimeInterval() { if (avgPostDataExecutionTimeInterval == -1) { long totalExecutionTime = 0; @@ -249,7 +248,7 @@ public final long getAvgPostDataExecutionTimeInterval() { } } if (totalAttempts > 0) { - avgPostDataExecutionTimeInterval = Math.round(totalExecutionTime*1.0/totalAttempts); + avgPostDataExecutionTimeInterval = Math.round(totalExecutionTime * 1.0 / totalAttempts); } } return avgPostDataExecutionTimeInterval; @@ -266,7 +265,7 @@ public final long getFinishTime() { public final long getInitTime() { return initTime; } - + public final long getInitRequestedTime() { return initRequestedTime; } @@ -274,7 +273,7 @@ public final long getInitRequestedTime() { public final long getStartRequestedTime() { return startRequestedTime; } - + @Override public final long getFinishTimeInterval() { long vertexEndTime = finishTime - (dagInfo.getStartTime()); @@ -296,7 +295,7 @@ public final String getDiagnostics() { public final String getVertexName() { return vertexName; } - + public final String getVertexId() { return vertexId; } @@ -342,10 +341,8 @@ public final int getNumFailedTaskAttemptsCount() { public final String getProcessorClassName() { return processorClass; - } - private List getTasksInternal() { return Lists.newLinkedList(taskInfoMap.values()); } @@ -410,7 +407,8 @@ public final List getSuccessfulTasks() { public final List getTasks(final TaskState state) { return Collections.unmodifiableList(Lists.newLinkedList(Iterables.filter(Lists.newLinkedList (taskInfoMap.values()), new Predicate() { - @Override public boolean apply(TaskInfo input) { + @Override + public boolean apply(TaskInfo input) { return input.getStatus() != null && input.getStatus().equals(state.toString()); } } @@ -471,7 +469,7 @@ public List getTaskAttempts() { * @return list of TaskAttemptInfo */ public final List getTaskAttempts(boolean sorted, - @Nullable Ordering ordering) { + @Nullable Ordering ordering) { List taskAttemptInfos = getTaskAttemptsInternal(); if (sorted) { Collections.sort(taskAttemptInfos, ((ordering == null) ? orderingOnAttemptStartTime() : ordering)); @@ -520,7 +518,8 @@ public final TaskInfo getFirstTaskToStart() { return null; } Collections.sort(taskInfoList, new Comparator() { - @Override public int compare(TaskInfo o1, TaskInfo o2) { + @Override + public int compare(TaskInfo o1, TaskInfo o2) { return Long.compare(o1.getStartTimeInterval(), o2.getStartTimeInterval()); } }); @@ -538,7 +537,8 @@ public final TaskInfo getLastTaskToFinish() { return null; } Collections.sort(taskInfoList, new Comparator() { - @Override public int compare(TaskInfo o1, TaskInfo o2) { + @Override + public int compare(TaskInfo o1, TaskInfo o2) { return -1 * Long.compare(o1.getFinishTimeInterval(), o2.getFinishTimeInterval()); } }); @@ -584,7 +584,8 @@ public final long getMaxTaskDuration() { private Ordering orderingOnTimeTaken() { return Ordering.from(new Comparator() { - @Override public int compare(TaskInfo o1, TaskInfo o2) { + @Override + public int compare(TaskInfo o1, TaskInfo o2) { return Long.compare(o1.getTimeTaken(), o2.getTimeTaken()); } }); @@ -592,7 +593,8 @@ private Ordering orderingOnTimeTaken() { private Ordering orderingOnStartTime() { return Ordering.from(new Comparator() { - @Override public int compare(TaskInfo o1, TaskInfo o2) { + @Override + public int compare(TaskInfo o1, TaskInfo o2) { return Long.compare(o1.getStartTimeInterval(), o2.getStartTimeInterval()); } }); @@ -600,7 +602,8 @@ private Ordering orderingOnStartTime() { private Ordering orderingOnAttemptStartTime() { return Ordering.from(new Comparator() { - @Override public int compare(TaskAttemptInfo o1, TaskAttemptInfo o2) { + @Override + public int compare(TaskAttemptInfo o1, TaskAttemptInfo o2) { return Long.compare(o1.getStartTimeInterval(), o2.getStartTimeInterval()); } }); diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/utils/Utils.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/utils/Utils.java index 94b50a6c5c..5c98ae15e1 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/utils/Utils.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/utils/Utils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -47,7 +47,6 @@ public class Utils { private static final String LOG4J_CONFIGURATION = "log4j.configuration"; private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(Utils.class); - /** * Parse tez counters from json * @@ -90,20 +89,20 @@ public static TezCounters parseTezCountersFromJSON(JSONObject jsonObject) } private static void addCounter(CounterGroup group, String counterName, String displayName, - long counterValue) { + long counterValue) { try { TezCounter counter = group.findCounter(counterName, displayName); counter.setValue(counterValue); - } catch(IllegalArgumentException e) { + } catch (IllegalArgumentException e) { LOG.debug("Error finding {} in {} with displayName {}", counterName, group, displayName); } } - - public static List parseDataEventDependencyFromJSON(JSONObject jsonObject) + + public static List parseDataEventDependencyFromJSON(JSONObject jsonObject) throws JSONException { List events = Lists.newArrayList(); JSONArray fields = jsonObject.optJSONArray(Constants.LAST_DATA_EVENTS); - for (int i=0; i + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -184,10 +184,8 @@ public static void setupTezCluster() throws Exception { tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSHistoryLoggingService.class.getName()); yarnTimelineAddress = miniTezCluster.getConfig().get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS); - } - /** * Run a word count example in mini cluster and check if it is possible to download * data from ATS and parse it. Also, run with SimpleHistoryLogging option and verify @@ -202,7 +200,7 @@ public void testParserWithSuccessfulJob() throws Exception { WordCount.SumProcessor.class.getName(), "WordCount", true); //Export the data from ATS - String[] args = { "--dagId=" + dagId, "--downloadDir=" + DOWNLOAD_DIR, "--yarnTimelineAddress=" + yarnTimelineAddress }; + String[] args = {"--dagId=" + dagId, "--downloadDir=" + DOWNLOAD_DIR, "--yarnTimelineAddress=" + yarnTimelineAddress}; int result = ATSImportTool.process(args); assertTrue(result == 0); @@ -309,7 +307,7 @@ private void verifyJobSpecificInfo(DagInfo dagInfo) { for (TaskAttemptInfo attempt : attempts) { DataDependencyEvent item = attempt.getLastDataEvents().get(0); assertTrue(item.getTimestamp() > 0); - + if (lastDataEventSourceTA == null) { lastDataEventSourceTA = item.getTaskAttemptId(); } else { @@ -350,20 +348,20 @@ private void verifyJobSpecificInfo(DagInfo dagInfo) { @Test public void testParserWithSuccessfulJob_InvalidATS() throws Exception { //Run basic word count example. - String dagId = runWordCount(WordCount.TokenProcessor.class.getName(), + String dagId = runWordCount(WordCount.TokenProcessor.class.getName(), WordCount.SumProcessor.class.getName(), "WordCount-With-WrongATS-URL", true); //Export the data from ATS String atsAddress = "--atsAddress=http://atsHost:8188"; - String[] args = { "--dagId=" + dagId, + String[] args = {"--dagId=" + dagId, "--downloadDir=" + DOWNLOAD_DIR, atsAddress - }; + }; try { int result = ATSImportTool.process(args); fail("Should have failed with processException"); - } catch(ParseException e) { + } catch (ParseException e) { //expects exception } } @@ -378,7 +376,7 @@ public void testParserWithFailedJob() throws Exception { .getName(), "WordCount-With-Exception", true); //Export the data from ATS - String[] args = { "--dagId=" + dagId, "--downloadDir=" + DOWNLOAD_DIR, "--yarnTimelineAddress=" + yarnTimelineAddress }; + String[] args = {"--dagId=" + dagId, "--downloadDir=" + DOWNLOAD_DIR, "--yarnTimelineAddress=" + yarnTimelineAddress}; int result = ATSImportTool.process(args); assertTrue(result == 0); @@ -435,7 +433,6 @@ public void testParserWithFailedJob() throws Exception { //Verify if the processor exception is given in diagnostics assertTrue(dagInfo.getDiagnostics().contains("Failing this processor for some reason")); - } /** @@ -559,7 +556,7 @@ private void isCounterSame(Map counter1, Map info1, - Collection info2) { + Collection info2) { assertTrue("sizes should be the same", info1.size() == info1.size()); Iterator it1 = info1.iterator(); Iterator it2 = info2.iterator(); @@ -582,7 +579,6 @@ private void isTaskAttemptEqual(TaskAttemptInfo info1, TaskAttemptInfo info2) { isCountersSame(info1, info2); } - /** * Create sample file for wordcount program * @@ -611,7 +607,7 @@ private DagInfo getDagInfo(String dagId) throws TezException { } private void verifyCounter(Map counterMap, - String counterGroupName, long expectedVal) { + String counterGroupName, long expectedVal) { //Iterate through group-->tezCounter for (Map.Entry entry : counterMap.entrySet()) { if (counterGroupName != null) { @@ -643,7 +639,7 @@ TezClient getTezClient(boolean withTimeline) throws Exception { } private String runWordCount(String tokenizerProcessor, String summationProcessor, - String dagName, boolean withTimeline) + String dagName, boolean withTimeline) throws Exception { //HDFS path Path outputLoc = new Path("/tmp/outPath_" + System.currentTimeMillis()); diff --git a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/DatePartitionedLogger.java b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/DatePartitionedLogger.java index a569567d1e..aaa867c937 100644 --- a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/DatePartitionedLogger.java +++ b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/DatePartitionedLogger.java @@ -49,10 +49,10 @@ public class DatePartitionedLogger { private static final Logger LOG = LoggerFactory.getLogger(DatePartitionedLogger.class); // Everyone has permission to write, but with sticky set so that delete is restricted. // This is required, since the path is same for all users and everyone writes into it. - private static final FsPermission DIR_PERMISSION = FsPermission.createImmutable((short)01777); + private static final FsPermission DIR_PERMISSION = FsPermission.createImmutable((short) 01777); // Since the directories have broad permissions restrict the file read access. - private static final FsPermission FILE_UMASK = FsPermission.createImmutable((short)0066); + private static final FsPermission FILE_UMASK = FsPermission.createImmutable((short) 0066); private final Parser parser; private final Path basePath; @@ -115,7 +115,7 @@ public Path getPathForSubdir(String dirName, String fileName) { */ public LocalDate getDateFromDir(String dirName) { if (!dirName.startsWith("date=")) { - throw new IllegalArgumentException("Invalid directory: "+ dirName); + throw new IllegalArgumentException("Invalid directory: " + dirName); } return LocalDate.parse(dirName.substring(5), DateTimeFormatter.ISO_LOCAL_DATE); } diff --git a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/HistoryEventProtoConverter.java b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/HistoryEventProtoConverter.java index f5f436eb9d..35d0b91657 100644 --- a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/HistoryEventProtoConverter.java +++ b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/HistoryEventProtoConverter.java @@ -77,51 +77,51 @@ public class HistoryEventProtoConverter { public HistoryEventProto convert(HistoryEvent historyEvent) { validateEvent(historyEvent); switch (historyEvent.getEventType()) { - case APP_LAUNCHED: - return convertAppLaunchedEvent((AppLaunchedEvent) historyEvent); - case AM_LAUNCHED: - return convertAMLaunchedEvent((AMLaunchedEvent) historyEvent); - case AM_STARTED: - return convertAMStartedEvent((AMStartedEvent) historyEvent); - case CONTAINER_LAUNCHED: - return convertContainerLaunchedEvent((ContainerLaunchedEvent) historyEvent); - case CONTAINER_STOPPED: - return convertContainerStoppedEvent((ContainerStoppedEvent) historyEvent); - case DAG_SUBMITTED: - return convertDAGSubmittedEvent((DAGSubmittedEvent) historyEvent); - case DAG_INITIALIZED: - return convertDAGInitializedEvent((DAGInitializedEvent) historyEvent); - case DAG_STARTED: - return convertDAGStartedEvent((DAGStartedEvent) historyEvent); - case DAG_FINISHED: - return convertDAGFinishedEvent((DAGFinishedEvent) historyEvent); - case VERTEX_INITIALIZED: - return convertVertexInitializedEvent((VertexInitializedEvent) historyEvent); - case VERTEX_STARTED: - return convertVertexStartedEvent((VertexStartedEvent) historyEvent); - case VERTEX_FINISHED: - return convertVertexFinishedEvent((VertexFinishedEvent) historyEvent); - case TASK_STARTED: - return convertTaskStartedEvent((TaskStartedEvent) historyEvent); - case TASK_FINISHED: - return convertTaskFinishedEvent((TaskFinishedEvent) historyEvent); - case TASK_ATTEMPT_STARTED: - return convertTaskAttemptStartedEvent((TaskAttemptStartedEvent) historyEvent); - case TASK_ATTEMPT_FINISHED: - return convertTaskAttemptFinishedEvent((TaskAttemptFinishedEvent) historyEvent); - case VERTEX_CONFIGURE_DONE: - return convertVertexReconfigureDoneEvent((VertexConfigurationDoneEvent) historyEvent); - case DAG_RECOVERED: - return convertDAGRecoveredEvent((DAGRecoveredEvent) historyEvent); - case VERTEX_COMMIT_STARTED: - case VERTEX_GROUP_COMMIT_STARTED: - case VERTEX_GROUP_COMMIT_FINISHED: - case DAG_COMMIT_STARTED: - case DAG_KILL_REQUEST: - throw new UnsupportedOperationException("Invalid Event, does not support history, eventType=" - + historyEvent.getEventType()); - // Do not add default, if a new event type is added, we'll get a warning for the - // switch. + case APP_LAUNCHED: + return convertAppLaunchedEvent((AppLaunchedEvent) historyEvent); + case AM_LAUNCHED: + return convertAMLaunchedEvent((AMLaunchedEvent) historyEvent); + case AM_STARTED: + return convertAMStartedEvent((AMStartedEvent) historyEvent); + case CONTAINER_LAUNCHED: + return convertContainerLaunchedEvent((ContainerLaunchedEvent) historyEvent); + case CONTAINER_STOPPED: + return convertContainerStoppedEvent((ContainerStoppedEvent) historyEvent); + case DAG_SUBMITTED: + return convertDAGSubmittedEvent((DAGSubmittedEvent) historyEvent); + case DAG_INITIALIZED: + return convertDAGInitializedEvent((DAGInitializedEvent) historyEvent); + case DAG_STARTED: + return convertDAGStartedEvent((DAGStartedEvent) historyEvent); + case DAG_FINISHED: + return convertDAGFinishedEvent((DAGFinishedEvent) historyEvent); + case VERTEX_INITIALIZED: + return convertVertexInitializedEvent((VertexInitializedEvent) historyEvent); + case VERTEX_STARTED: + return convertVertexStartedEvent((VertexStartedEvent) historyEvent); + case VERTEX_FINISHED: + return convertVertexFinishedEvent((VertexFinishedEvent) historyEvent); + case TASK_STARTED: + return convertTaskStartedEvent((TaskStartedEvent) historyEvent); + case TASK_FINISHED: + return convertTaskFinishedEvent((TaskFinishedEvent) historyEvent); + case TASK_ATTEMPT_STARTED: + return convertTaskAttemptStartedEvent((TaskAttemptStartedEvent) historyEvent); + case TASK_ATTEMPT_FINISHED: + return convertTaskAttemptFinishedEvent((TaskAttemptFinishedEvent) historyEvent); + case VERTEX_CONFIGURE_DONE: + return convertVertexReconfigureDoneEvent((VertexConfigurationDoneEvent) historyEvent); + case DAG_RECOVERED: + return convertDAGRecoveredEvent((DAGRecoveredEvent) historyEvent); + case VERTEX_COMMIT_STARTED: + case VERTEX_GROUP_COMMIT_STARTED: + case VERTEX_GROUP_COMMIT_FINISHED: + case DAG_COMMIT_STARTED: + case DAG_KILL_REQUEST: + throw new UnsupportedOperationException("Invalid Event, does not support history, eventType=" + + historyEvent.getEventType()); + // Do not add default, if a new event type is added, we'll get a warning for the + // switch. } throw new UnsupportedOperationException( "Unhandled Event, eventType=" + historyEvent.getEventType()); @@ -135,8 +135,8 @@ private void validateEvent(HistoryEvent event) { } private HistoryEventProto.Builder makeBuilderForEvent(HistoryEvent event, long time, - TezDAGID dagId, ApplicationId appId, ApplicationAttemptId appAttemptId, TezVertexID vertexId, - TezTaskID taskId, TezTaskAttemptID taskAttemptId, String user) { + TezDAGID dagId, ApplicationId appId, ApplicationAttemptId appAttemptId, TezVertexID vertexId, + TezTaskID taskId, TezTaskAttemptID taskAttemptId, String user) { HistoryEventProto.Builder builder = HistoryEventProto.newBuilder(); builder.setEventType(event.getEventType().name()); builder.setEventTime(time); @@ -185,7 +185,7 @@ private void addEventData(HistoryEventProto.Builder builder, String key, Number } private void addEventData(HistoryEventProto.Builder builder, String key, - Map value) { + Map value) { try { builder.addEventData( KVPair.newBuilder().setKey(key).setValue(mapper.writeValueAsString(value))); @@ -198,8 +198,8 @@ private HistoryEventProto convertAppLaunchedEvent(AppLaunchedEvent event) { HistoryEventProto.Builder builder = makeBuilderForEvent(event, event.getLaunchTime(), null, event.getApplicationId(), null, null, null, null, event.getUser()); // This is ok as long as we do not modify the underlying map. - @SuppressWarnings({ "unchecked", "rawtypes" }) - Map confMap = (Map)DAGUtils.convertConfigurationToATSMap(event.getConf()); + @SuppressWarnings({"unchecked", "rawtypes"}) + Map confMap = (Map) DAGUtils.convertConfigurationToATSMap(event.getConf()); addEventData(builder, ATSConstants.CONFIG, confMap); if (event.getVersion() != null) { addEventData(builder, ATSConstants.TEZ_VERSION, diff --git a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/HistoryEventProtoJsonConversion.java b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/HistoryEventProtoJsonConversion.java index ef84b2ec61..d659b9e460 100644 --- a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/HistoryEventProtoJsonConversion.java +++ b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/HistoryEventProtoJsonConversion.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -45,69 +45,69 @@ public static JSONObject convertToJson(HistoryEventProto historyEvent) throws JS JSONObject jsonObject = null; switch (historyEvent.getEventType()) { - case "APP_LAUNCHED": - jsonObject = convertAppLaunchedEvent(historyEvent); - break; - case "AM_LAUNCHED": - jsonObject = convertAMLaunchedEvent(historyEvent); - break; - case "AM_STARTED": - jsonObject = convertAMStartedEvent(historyEvent); - break; - case "CONTAINER_LAUNCHED": - jsonObject = convertContainerLaunchedEvent(historyEvent); - break; - case "CONTAINER_STOPPED": - jsonObject = convertContainerStoppedEvent(historyEvent); - break; - case "DAG_SUBMITTED": - jsonObject = convertDAGSubmittedEvent(historyEvent); - break; - case "DAG_INITIALIZED": - jsonObject = convertDAGInitializedEvent(historyEvent); - break; - case "DAG_STARTED": - jsonObject = convertDAGStartedEvent(historyEvent); - break; - case "DAG_FINISHED": - jsonObject = convertDAGFinishedEvent(historyEvent); - break; - case "VERTEX_INITIALIZED": - jsonObject = convertVertexInitializedEvent(historyEvent); - break; - case "VERTEX_STARTED": - jsonObject = convertVertexStartedEvent(historyEvent); - break; - case "VERTEX_FINISHED": - jsonObject = convertVertexFinishedEvent(historyEvent); - break; - case "TASK_STARTED": - jsonObject = convertTaskStartedEvent(historyEvent); - break; - case "TASK_FINISHED": - jsonObject = convertTaskFinishedEvent(historyEvent); - break; - case "TASK_ATTEMPT_STARTED": - jsonObject = convertTaskAttemptStartedEvent(historyEvent); - break; - case "TASK_ATTEMPT_FINISHED": - jsonObject = convertTaskAttemptFinishedEvent(historyEvent); - break; - case "VERTEX_CONFIGURE_DONE": - jsonObject = convertVertexReconfigureDoneEvent(historyEvent); - break; - case "DAG_RECOVERED": - jsonObject = convertDAGRecoveredEvent(historyEvent); - break; - case "VERTEX_COMMIT_STARTED": - case "VERTEX_GROUP_COMMIT_STARTED": - case "VERTEX_GROUP_COMMIT_FINISHED": - case "DAG_COMMIT_STARTED": - throw new UnsupportedOperationException( - "Invalid Event, does not support history" + ", eventType=" + historyEvent.getEventType()); - default: - throw new UnsupportedOperationException( - "Unhandled Event" + ", eventType=" + historyEvent.getEventType()); + case "APP_LAUNCHED": + jsonObject = convertAppLaunchedEvent(historyEvent); + break; + case "AM_LAUNCHED": + jsonObject = convertAMLaunchedEvent(historyEvent); + break; + case "AM_STARTED": + jsonObject = convertAMStartedEvent(historyEvent); + break; + case "CONTAINER_LAUNCHED": + jsonObject = convertContainerLaunchedEvent(historyEvent); + break; + case "CONTAINER_STOPPED": + jsonObject = convertContainerStoppedEvent(historyEvent); + break; + case "DAG_SUBMITTED": + jsonObject = convertDAGSubmittedEvent(historyEvent); + break; + case "DAG_INITIALIZED": + jsonObject = convertDAGInitializedEvent(historyEvent); + break; + case "DAG_STARTED": + jsonObject = convertDAGStartedEvent(historyEvent); + break; + case "DAG_FINISHED": + jsonObject = convertDAGFinishedEvent(historyEvent); + break; + case "VERTEX_INITIALIZED": + jsonObject = convertVertexInitializedEvent(historyEvent); + break; + case "VERTEX_STARTED": + jsonObject = convertVertexStartedEvent(historyEvent); + break; + case "VERTEX_FINISHED": + jsonObject = convertVertexFinishedEvent(historyEvent); + break; + case "TASK_STARTED": + jsonObject = convertTaskStartedEvent(historyEvent); + break; + case "TASK_FINISHED": + jsonObject = convertTaskFinishedEvent(historyEvent); + break; + case "TASK_ATTEMPT_STARTED": + jsonObject = convertTaskAttemptStartedEvent(historyEvent); + break; + case "TASK_ATTEMPT_FINISHED": + jsonObject = convertTaskAttemptFinishedEvent(historyEvent); + break; + case "VERTEX_CONFIGURE_DONE": + jsonObject = convertVertexReconfigureDoneEvent(historyEvent); + break; + case "DAG_RECOVERED": + jsonObject = convertDAGRecoveredEvent(historyEvent); + break; + case "VERTEX_COMMIT_STARTED": + case "VERTEX_GROUP_COMMIT_STARTED": + case "VERTEX_GROUP_COMMIT_FINISHED": + case "DAG_COMMIT_STARTED": + throw new UnsupportedOperationException( + "Invalid Event, does not support history" + ", eventType=" + historyEvent.getEventType()); + default: + throw new UnsupportedOperationException( + "Unhandled Event" + ", eventType=" + historyEvent.getEventType()); } return jsonObject; } diff --git a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoHistoryLoggingService.java b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoHistoryLoggingService.java index d675a659a0..94c536320c 100644 --- a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoHistoryLoggingService.java +++ b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoHistoryLoggingService.java @@ -170,7 +170,7 @@ private void handleEvent(DAGHistoryEvent event) throws IOException { HistoryEventType type = historyEvent.getEventType(); TezDAGID dagId = event.getDAGID(); if (type == HistoryEventType.DAG_FINISHED) { - finishCurrentDag((DAGFinishedEvent)historyEvent); + finishCurrentDag((DAGFinishedEvent) historyEvent); } else if (type == HistoryEventType.DAG_SUBMITTED) { finishCurrentDag(null); currentDagId = dagId; diff --git a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageWritable.java b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageWritable.java index 34e47014e1..cedb195d83 100644 --- a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageWritable.java +++ b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/ProtoMessageWritable.java @@ -54,6 +54,7 @@ public void setMessage(T message) { private static class DataOutputStream extends OutputStream { DataOutput out; + @Override public void write(int b) throws IOException { out.write(b); @@ -78,6 +79,7 @@ public void write(DataOutput out) throws IOException { private static class DataInputStream extends InputStream { DataInput in; + @Override public int read() throws IOException { try { diff --git a/tez-plugins/tez-protobuf-history-plugin/src/test/java/org/apache/tez/dag/history/logging/proto/TestDagManifestFileScanner.java b/tez-plugins/tez-protobuf-history-plugin/src/test/java/org/apache/tez/dag/history/logging/proto/TestDagManifestFileScanner.java index 4950522429..9ad267491c 100644 --- a/tez-plugins/tez-protobuf-history-plugin/src/test/java/org/apache/tez/dag/history/logging/proto/TestDagManifestFileScanner.java +++ b/tez-plugins/tez-protobuf-history-plugin/src/test/java/org/apache/tez/dag/history/logging/proto/TestDagManifestFileScanner.java @@ -53,7 +53,7 @@ public void setupTest() throws Exception { manifestLogger = loggers.getManifestEventsLogger(); } - @Test(timeout=5000) + @Test(timeout = 5000) public void testNormal() throws Exception { clock.setTime(0); // 0th day. createManifestEvents(0, 8); @@ -91,7 +91,8 @@ public void testNormal() throws Exception { } private Path deleteFilePath = null; - @Test(timeout=5000) + + @Test(timeout = 5000) public void testError() throws Exception { clock.setTime(0); // 0th day. createManifestEvents(0, 4); diff --git a/tez-plugins/tez-protobuf-history-plugin/src/test/java/org/apache/tez/dag/history/logging/proto/TestHistoryEventProtoConverter.java b/tez-plugins/tez-protobuf-history-plugin/src/test/java/org/apache/tez/dag/history/logging/proto/TestHistoryEventProtoConverter.java index 64f66bc12b..c4ba13a369 100644 --- a/tez-plugins/tez-protobuf-history-plugin/src/test/java/org/apache/tez/dag/history/logging/proto/TestHistoryEventProtoConverter.java +++ b/tez-plugins/tez-protobuf-history-plugin/src/test/java/org/apache/tez/dag/history/logging/proto/TestHistoryEventProtoConverter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -257,7 +257,7 @@ private String safeToString(Object obj) { } private void assertCommon(HistoryEventProto proto, HistoryEventType type, long eventTime, - EntityTypes entityType, ApplicationAttemptId appAttemptId, String user, int numData) { + EntityTypes entityType, ApplicationAttemptId appAttemptId, String user, int numData) { Assert.assertEquals(type.name(), proto.getEventType()); Assert.assertEquals(eventTime, proto.getEventTime()); // Assert.assertEquals(safeToString(appId), proto.getAppId()); @@ -265,19 +265,19 @@ private void assertCommon(HistoryEventProto proto, HistoryEventType type, long e Assert.assertEquals(safeToString(user), proto.getUser()); if (entityType != null) { switch (entityType) { // Intentional fallthrough. - case TEZ_TASK_ATTEMPT_ID: - Assert.assertEquals(tezTaskAttemptID.toString(), proto.getTaskAttemptId()); - case TEZ_TASK_ID: - Assert.assertEquals(tezTaskID.toString(), proto.getTaskId()); - case TEZ_VERTEX_ID: - Assert.assertEquals(tezVertexID.toString(), proto.getVertexId()); - case TEZ_DAG_ID: - Assert.assertEquals(tezDAGID.toString(), proto.getDagId()); - case TEZ_APPLICATION: - Assert.assertEquals(applicationId.toString(), proto.getAppId()); - break; - default: - Assert.fail("Invalid type: " + entityType.name()); + case TEZ_TASK_ATTEMPT_ID: + Assert.assertEquals(tezTaskAttemptID.toString(), proto.getTaskAttemptId()); + case TEZ_TASK_ID: + Assert.assertEquals(tezTaskID.toString(), proto.getTaskId()); + case TEZ_VERTEX_ID: + Assert.assertEquals(tezVertexID.toString(), proto.getVertexId()); + case TEZ_DAG_ID: + Assert.assertEquals(tezDAGID.toString(), proto.getDagId()); + case TEZ_APPLICATION: + Assert.assertEquals(applicationId.toString(), proto.getAppId()); + break; + default: + Assert.fail("Invalid type: " + entityType.name()); } } Assert.assertEquals(numData, proto.getEventDataCount()); @@ -661,7 +661,7 @@ public void testConvertVertexReconfigreDoneEvent() { new HashMap(); edgeMgrs.put("a", EdgeProperty.create(EdgeManagerPluginDescriptor.create("a.class") - .setHistoryText("text"), DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, + .setHistoryText("text"), DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"))); VertexConfigurationDoneEvent event = new VertexConfigurationDoneEvent(vId, 0L, 1, null, edgeMgrs, null, true); diff --git a/tez-plugins/tez-protobuf-history-plugin/src/test/java/org/apache/tez/dag/history/logging/proto/TestProtoHistoryLoggingService.java b/tez-plugins/tez-protobuf-history-plugin/src/test/java/org/apache/tez/dag/history/logging/proto/TestProtoHistoryLoggingService.java index fd3154d904..f8b3d055df 100644 --- a/tez-plugins/tez-protobuf-history-plugin/src/test/java/org/apache/tez/dag/history/logging/proto/TestProtoHistoryLoggingService.java +++ b/tez-plugins/tez-protobuf-history-plugin/src/test/java/org/apache/tez/dag/history/logging/proto/TestProtoHistoryLoggingService.java @@ -221,14 +221,15 @@ public void testServiceSplitEvents() throws Exception { } private List makeHistoryEvents(TezDAGID dagId, - ProtoHistoryLoggingService service) { + ProtoHistoryLoggingService service) { List historyEvents = new ArrayList<>(); DAGPlan dagPlan = DAGPlan.newBuilder().setName("DAGPlanMock").build(); long time = System.currentTimeMillis(); Configuration conf = new Configuration(service.getConfig()); historyEvents.add(new DAGHistoryEvent(null, new AppLaunchedEvent(appId, time, time, user, conf, - new VersionInfo("component", "1.1.0", "rev1", "20120101", "git.apache.org") {}))); + new VersionInfo("component", "1.1.0", "rev1", "20120101", "git.apache.org") { + }))); historyEvents.add(new DAGHistoryEvent(dagId, new DAGSubmittedEvent(dagId, time, DAGPlan.getDefaultInstance(), attemptId, null, user, conf, null, "default"))); historyEvents.add(new DAGHistoryEvent(dagId, new DAGInitializedEvent(dagId, time + 1, user, @@ -272,7 +273,8 @@ private ProtoHistoryLoggingService createService(boolean splitEvents) throws IOE when(appContext.getApplicationID()).thenReturn(appId); when(appContext.getApplicationAttemptId()).thenReturn(attemptId); when(appContext.getUser()).thenReturn(user); - when(appContext.getHadoopShim()).thenReturn(new HadoopShim() {}); + when(appContext.getHadoopShim()).thenReturn(new HadoopShim() { + }); when(appContext.getClock()).thenReturn(clock); service.setAppContext(appContext); Configuration conf = new Configuration(false); @@ -285,7 +287,7 @@ private ProtoHistoryLoggingService createService(boolean splitEvents) throws IOE } private void assertEventsRead(ProtoMessageReader reader, - List protos, int start, int finish) throws Exception { + List protos, int start, int finish) throws Exception { for (int i = start; i < finish; ++i) { try { HistoryEventProto evt = reader.readEvent(); diff --git a/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java b/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java index 2ee2c21b79..bf70856c13 100644 --- a/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java +++ b/tez-plugins/tez-yarn-timeline-cache-plugin/src/main/java/org/apache/tez/dag/history/logging/ats/TimelineCachePluginImpl.java @@ -96,7 +96,6 @@ private Set convertToTimelineEntityGroupIds(String entity if (vertexID != null) { return createTimelineEntityGroupIds(vertexID.getDAGID()); } - } else if (entityType.equals(EntityTypes.TEZ_TASK_ID.name())) { TezTaskID taskID = TezTaskID.fromString(entityId); if (taskID != null) { @@ -124,8 +123,8 @@ private Set convertToTimelineEntityGroupIds(String entity @Override public Set getTimelineEntityGroupId(String entityType, - NameValuePair primaryFilter, - Collection secondaryFilters) { + NameValuePair primaryFilter, + Collection secondaryFilters) { if (!knownEntityTypes.contains(entityType) || primaryFilter == null || !knownEntityTypes.contains(primaryFilter.getName()) @@ -145,7 +144,7 @@ public Set getTimelineEntityGroupId(String entityId, Stri @Override public Set getTimelineEntityGroupId(String entityType, - SortedSet entityIds, Set eventTypes) { + SortedSet entityIds, Set eventTypes) { if (!knownEntityTypes.contains(entityType) || summaryEntityTypes.contains(entityType) || entityIds == null || entityIds.isEmpty()) { @@ -180,7 +179,7 @@ private Set loadAllNumDagsPerGroup() { } // Add the older values from config. - int [] usedNumGroups = conf.getInts(TezConfiguration.TEZ_HISTORY_LOGGING_TIMELINE_CACHE_PLUGIN_OLD_NUM_DAGS_PER_GROUP); + int[] usedNumGroups = conf.getInts(TezConfiguration.TEZ_HISTORY_LOGGING_TIMELINE_CACHE_PLUGIN_OLD_NUM_DAGS_PER_GROUP); if (usedNumGroups != null) { for (int i = 0; i < usedNumGroups.length; ++i) { allNumDagsPerGroup.add(usedNumGroups[i]); @@ -199,5 +198,4 @@ private Set loadAllNumDagsPerGroup() { public Configuration getConf() { return conf; } - } diff --git a/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java b/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java index 1bfa0a143a..7d291944f1 100644 --- a/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java +++ b/tez-plugins/tez-yarn-timeline-cache-plugin/src/test/java/org/apache/tez/dag/history/logging/ats/TestTimelineCachePluginImpl.java @@ -46,7 +46,6 @@ import com.google.common.collect.Sets; - public class TestTimelineCachePluginImpl { static ApplicationId appId1; @@ -321,7 +320,6 @@ public void testInvalidTypeRequests() { new HashSet())); Assert.assertNull(plugin.getTimelineEntityGroupId(EntityTypes.TEZ_TASK_ID.name(), null, new HashSet())); - } @Test @@ -377,7 +375,7 @@ public void testContainerIdConversion() { Assert.assertEquals("All groupIds not returned", 1, found); } - private Set getGroupIds(TezDAGID dagId, int ... allNumDagsPerGroup) { + private Set getGroupIds(TezDAGID dagId, int... allNumDagsPerGroup) { HashSet groupIds = Sets.newHashSet(dagId.toString()); for (int numDagsPerGroup : allNumDagsPerGroup) { groupIds.add(dagId.getGroupId(numDagsPerGroup)); diff --git a/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/ats/acls/ATSHistoryACLPolicyManager.java b/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/ats/acls/ATSHistoryACLPolicyManager.java index 45e24ce72d..09a47c1d34 100644 --- a/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/ats/acls/ATSHistoryACLPolicyManager.java +++ b/tez-plugins/tez-yarn-timeline-history-with-acls/src/main/java/org/apache/tez/dag/history/ats/acls/ATSHistoryACLPolicyManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -63,14 +63,14 @@ private void initializeTimelineClient() { this.timelineClient = null; } if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { this.timelineClient = TimelineClient.createTimelineClient(); this.timelineClient.init(this.conf); this.timelineClient.start(); } else { this.timelineClient = null; if (conf.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "") - .equals(atsHistoryLoggingServiceClassName)) { + .equals(atsHistoryLoggingServiceClassName)) { LOG.warn(atsHistoryLoggingServiceClassName + " is disabled due to Timeline Service being disabled, " + YarnConfiguration.TIMELINE_SERVICE_ENABLED + " set to false"); @@ -84,7 +84,7 @@ private void initializeTimelineClient() { } private String getMergedViewACLs(ACLConfigurationParser parser, - DAGAccessControls dagAccessControls) { + DAGAccessControls dagAccessControls) { Map> allowedUsers = parser.getAllowedUsers(); Map> allowedGroups = parser.getAllowedGroups(); @@ -114,7 +114,7 @@ private String getMergedViewACLs(ACLConfigurationParser parser, } private void createTimelineDomain(String domainId, Configuration tezConf, - DAGAccessControls dagAccessControls) throws IOException, HistoryACLPolicyException { + DAGAccessControls dagAccessControls) throws IOException, HistoryACLPolicyException { TimelineDomain timelineDomain = new TimelineDomain(); timelineDomain.setId(domainId); @@ -129,13 +129,12 @@ private void createTimelineDomain(String domainId, Configuration tezConf, } catch (Exception e) { LOG.warn("Could not post timeline domain", e); throw new - HistoryACLPolicyException("Fail to create ACL-related domain in Timeline", e); + HistoryACLPolicyException("Fail to create ACL-related domain in Timeline", e); } } - private Map createSessionDomain(Configuration tezConf, - ApplicationId applicationId, DAGAccessControls dagAccessControls) + ApplicationId applicationId, DAGAccessControls dagAccessControls) throws IOException, HistoryACLPolicyException { String domainId = tezConf.get(TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID); @@ -168,7 +167,7 @@ private Map createSessionDomain(Configuration tezConf, } private Map createDAGDomain(Configuration tezConf, - ApplicationId applicationId, String dagName, DAGAccessControls dagAccessControls) + ApplicationId applicationId, String dagName, DAGAccessControls dagAccessControls) throws IOException, HistoryACLPolicyException { String domainId = tezConf.get(TezConfiguration.YARN_ATS_ACL_DAG_DOMAIN_ID); @@ -224,13 +223,13 @@ public Map setupSessionACLs(Configuration conf, ApplicationId ap @Override public Map setupNonSessionACLs(Configuration conf, ApplicationId applicationId, - DAGAccessControls dagAccessControls) throws IOException, HistoryACLPolicyException { + DAGAccessControls dagAccessControls) throws IOException, HistoryACLPolicyException { return createSessionDomain(conf, applicationId, dagAccessControls); } @Override public Map setupSessionDAGACLs(Configuration conf, ApplicationId applicationId, - String dagName, DAGAccessControls dagAccessControls) throws IOException, HistoryACLPolicyException { + String dagName, DAGAccessControls dagAccessControls) throws IOException, HistoryACLPolicyException { return createDAGDomain(conf, applicationId, dagName, dagAccessControls); } @@ -250,5 +249,4 @@ public void close() { timelineClient.stop(); } } - } diff --git a/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java b/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java index f284fe4049..7622fa0489 100644 --- a/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java +++ b/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -79,7 +79,6 @@ import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.WebResource; - public class TestATSHistoryWithACLs { private static final Logger LOG = LoggerFactory.getLogger(TestATSHistoryWithACLs.class); @@ -216,11 +215,11 @@ private TimelineDomain getDomain(String domainId) { } private void verifyDomainACLs(TimelineDomain timelineDomain, - Collection users, Collection groups) { + Collection users, Collection groups) { String readers = timelineDomain.getReaders(); int pos = readers.indexOf(" "); String readerUsers = readers.substring(0, pos); - String readerGroups = readers.substring(pos+1); + String readerGroups = readers.substring(pos + 1); assertTrue(readerUsers.contains(user)); for (String s : users) { @@ -233,10 +232,9 @@ private void verifyDomainACLs(TimelineDomain timelineDomain, if (!user.equals("nobody1") && !users.contains("nobody1")) { assertFalse(readerUsers.contains("nobody1")); } - } - @Test (timeout=50000) + @Test(timeout = 50000) public void testSimpleAMACls() throws Exception { TezClient tezSession = null; ApplicationId applicationId; @@ -288,7 +286,7 @@ public void testSimpleAMACls() throws Exception { verifyEntityDomains(applicationId, true); } - @Test (timeout=50000) + @Test(timeout = 50000) public void testDAGACls() throws Exception { TezClient tezSession = null; ApplicationId applicationId; @@ -355,7 +353,7 @@ public void testDAGACls() throws Exception { * due to failure to create domain in session start * @throws Exception */ - @Test (timeout=50000) + @Test(timeout = 50000) public void testDisableSessionLogging() throws Exception { TezClient tezSession = null; String viewAcls = "nobody nobody_group"; @@ -364,7 +362,7 @@ public void testDisableSessionLogging() throws Exception { DAG dag = DAG.create("TezSleepProcessor"); Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create( SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1, - Resource.newInstance(256, 1)); + Resource.newInstance(256, 1)); dag.addVertex(vertex); DAGAccessControls accessControls = new DAGAccessControls(); accessControls.setUsersWithViewACLs(Collections.singleton("nobody2")); @@ -397,8 +395,8 @@ public void testDisableSessionLogging() throws Exception { //////submit second dag////// DAG dag2 = DAG.create("TezSleepProcessor2"); vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create( - SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1, - Resource.newInstance(256, 1)); + SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1, + Resource.newInstance(256, 1)); dag2.addVertex(vertex); accessControls = new DAGAccessControls(); accessControls.setUsersWithViewACLs(Collections.singleton("nobody3")); @@ -408,7 +406,7 @@ public void testDisableSessionLogging() throws Exception { dagStatus = dagClient.getDAGStatus(null); while (!dagStatus.isCompleted()) { LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: " - + dagStatus.getState()); + + dagStatus.getState()); Thread.sleep(500l); dagStatus = dagClient.getDAGStatus(null); } @@ -420,7 +418,7 @@ public void testDisableSessionLogging() throws Exception { * in dagsubmittedevent is set off * @throws Exception */ - @Test (timeout=50000) + @Test(timeout = 50000) public void testDagLoggingDisabled() throws Exception { ATSHistoryLoggingService historyLoggingService; historyLoggingService = @@ -441,17 +439,17 @@ public void testDagLoggingDisabled() throws Exception { historyLoggingService.start(); ApplicationId appId = ApplicationId.newInstance(100l, 1); TezDAGID tezDAGID = TezDAGID.getInstance( - appId, 100); + appId, 100); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); DAGPlan dagPlan = DAGPlan.newBuilder().setName("DAGPlanMock").build(); DAGSubmittedEvent submittedEvent = new DAGSubmittedEvent(tezDAGID, - 1, dagPlan, appAttemptId, null, - "usr", tezConf, null, null); + 1, dagPlan, appAttemptId, null, + "usr", tezConf, null, null); submittedEvent.setHistoryLoggingEnabled(false); DAGHistoryEvent event = new DAGHistoryEvent(tezDAGID, submittedEvent); historyLoggingService.handle(new DAGHistoryEvent(tezDAGID, submittedEvent)); Thread.sleep(1000l); - String url = "http://" + timelineAddress + "/ws/v1/timeline/TEZ_DAG_ID/"+event.getDAGID(); + String url = "http://" + timelineAddress + "/ws/v1/timeline/TEZ_DAG_ID/" + event.getDAGID(); Client client = new Client(); WebResource resource = client.resource(url); @@ -459,17 +457,17 @@ public void testDagLoggingDisabled() throws Exception { .get(ClientResponse.class); assertEquals(404, response.getStatus()); } - + /** * use mini cluster to verify data do push to ats when * the dag logging flag in dagsubmitted event is set on * @throws Exception */ - @Test (timeout=50000) + @Test(timeout = 50000) public void testDagLoggingEnabled() throws Exception { ATSHistoryLoggingService historyLoggingService; historyLoggingService = - ReflectionUtils.createClazzInstance(ATSHistoryLoggingService.class.getName()); + ReflectionUtils.createClazzInstance(ATSHistoryLoggingService.class.getName()); AppContext appContext = mock(AppContext.class); when(appContext.getApplicationID()).thenReturn(ApplicationId.newInstance(0, 1)); historyLoggingService.setAppContext(appContext); @@ -486,17 +484,17 @@ public void testDagLoggingEnabled() throws Exception { historyLoggingService.start(); ApplicationId appId = ApplicationId.newInstance(100l, 1); TezDAGID tezDAGID = TezDAGID.getInstance( - appId, 11); + appId, 11); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); DAGPlan dagPlan = DAGPlan.newBuilder().setName("DAGPlanMock").build(); DAGSubmittedEvent submittedEvent = new DAGSubmittedEvent(tezDAGID, - 1, dagPlan, appAttemptId, null, - "usr", tezConf, null, null); + 1, dagPlan, appAttemptId, null, + "usr", tezConf, null, null); submittedEvent.setHistoryLoggingEnabled(true); DAGHistoryEvent event = new DAGHistoryEvent(tezDAGID, submittedEvent); historyLoggingService.handle(new DAGHistoryEvent(tezDAGID, submittedEvent)); Thread.sleep(1000l); - String url = "http://" + timelineAddress + "/ws/v1/timeline/TEZ_DAG_ID/"+event.getDAGID(); + String url = "http://" + timelineAddress + "/ws/v1/timeline/TEZ_DAG_ID/" + event.getDAGID(); Client client = new Client(); WebResource resource = client.resource(url); @@ -512,12 +510,13 @@ public void testDagLoggingEnabled() throws Exception { private static final String atsHistoryACLManagerClassName = "org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager"; - @Test (timeout=50000) + + @Test(timeout = 50000) public void testTimelineServiceDisabled() throws Exception { TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig()); tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, ATSHistoryLoggingService.class.getName()); - tezConf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,false); + tezConf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false); ATSHistoryACLPolicyManager historyACLPolicyManager = ReflectionUtils.createClazzInstance( atsHistoryACLManagerClassName); historyACLPolicyManager.setConf(tezConf); @@ -548,5 +547,4 @@ private void verifyEntityDomains(ApplicationId applicationId, boolean sameDomain assertEquals(appEntity.getDomainId(), dagEntity.getDomainId()); } } - } diff --git a/tez-plugins/tez-yarn-timeline-history-with-fs/src/main/java/org/apache/tez/dag/history/ats/acls/ATSV15HistoryACLPolicyManager.java b/tez-plugins/tez-yarn-timeline-history-with-fs/src/main/java/org/apache/tez/dag/history/ats/acls/ATSV15HistoryACLPolicyManager.java index d28ffe09a6..083bd49b19 100644 --- a/tez-plugins/tez-yarn-timeline-history-with-fs/src/main/java/org/apache/tez/dag/history/ats/acls/ATSV15HistoryACLPolicyManager.java +++ b/tez-plugins/tez-yarn-timeline-history-with-fs/src/main/java/org/apache/tez/dag/history/ats/acls/ATSV15HistoryACLPolicyManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -64,14 +64,14 @@ private void initializeTimelineClient() { this.timelineClient = null; } if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { this.timelineClient = TimelineClient.createTimelineClient(); this.timelineClient.init(this.conf); this.timelineClient.start(); } else { this.timelineClient = null; if (conf.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "") - .equals(atsHistoryLoggingServiceClassName)) { + .equals(atsHistoryLoggingServiceClassName)) { LOG.warn(atsHistoryLoggingServiceClassName + " is disabled due to Timeline Service being disabled, " + YarnConfiguration.TIMELINE_SERVICE_ENABLED + " set to false"); @@ -85,7 +85,7 @@ private void initializeTimelineClient() { } private String getMergedViewACLs(ACLConfigurationParser parser, - DAGAccessControls dagAccessControls) { + DAGAccessControls dagAccessControls) { Map> allowedUsers = parser.getAllowedUsers(); Map> allowedGroups = parser.getAllowedGroups(); @@ -115,7 +115,7 @@ private String getMergedViewACLs(ACLConfigurationParser parser, } private void createTimelineDomain(ApplicationId applicationId, - String domainId, Configuration tezConf, DAGAccessControls dagAccessControls) + String domainId, Configuration tezConf, DAGAccessControls dagAccessControls) throws IOException, HistoryACLPolicyException { TimelineDomain timelineDomain = new TimelineDomain(); timelineDomain.setId(domainId); @@ -133,13 +133,12 @@ private void createTimelineDomain(ApplicationId applicationId, } catch (Exception e) { LOG.warn("Could not post timeline domain", e); throw new - HistoryACLPolicyException("Fail to create ACL-related domain in Timeline", e); + HistoryACLPolicyException("Fail to create ACL-related domain in Timeline", e); } } - private Map createSessionDomain(Configuration tezConf, - ApplicationId applicationId, DAGAccessControls dagAccessControls) + ApplicationId applicationId, DAGAccessControls dagAccessControls) throws IOException, HistoryACLPolicyException { String domainId = tezConf.get(TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID); @@ -172,7 +171,7 @@ private Map createSessionDomain(Configuration tezConf, } private Map createDAGDomain(Configuration tezConf, - ApplicationId applicationId, String dagName, DAGAccessControls dagAccessControls) + ApplicationId applicationId, String dagName, DAGAccessControls dagAccessControls) throws IOException, HistoryACLPolicyException { String domainId = tezConf.get(TezConfiguration.YARN_ATS_ACL_DAG_DOMAIN_ID); @@ -228,13 +227,13 @@ public Map setupSessionACLs(Configuration conf, ApplicationId ap @Override public Map setupNonSessionACLs(Configuration conf, ApplicationId applicationId, - DAGAccessControls dagAccessControls) throws IOException, HistoryACLPolicyException { + DAGAccessControls dagAccessControls) throws IOException, HistoryACLPolicyException { return createSessionDomain(conf, applicationId, dagAccessControls); } @Override public Map setupSessionDAGACLs(Configuration conf, ApplicationId applicationId, - String dagName, DAGAccessControls dagAccessControls) throws IOException, HistoryACLPolicyException { + String dagName, DAGAccessControls dagAccessControls) throws IOException, HistoryACLPolicyException { return createDAGDomain(conf, applicationId, dagName, dagAccessControls); } @@ -254,5 +253,4 @@ public void close() { timelineClient.stop(); } } - } diff --git a/tez-plugins/tez-yarn-timeline-history-with-fs/src/main/java/org/apache/tez/dag/history/logging/ats/ATSV15HistoryLoggingService.java b/tez-plugins/tez-yarn-timeline-history-with-fs/src/main/java/org/apache/tez/dag/history/logging/ats/ATSV15HistoryLoggingService.java index 5068fb77db..9f1f8a1d8a 100644 --- a/tez-plugins/tez-yarn-timeline-history-with-fs/src/main/java/org/apache/tez/dag/history/logging/ats/ATSV15HistoryLoggingService.java +++ b/tez-plugins/tez-yarn-timeline-history-with-fs/src/main/java/org/apache/tez/dag/history/logging/ats/ATSV15HistoryLoggingService.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -121,13 +121,13 @@ public void serviceInit(Configuration serviceConf) throws Exception { } if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { timelineClient = TimelineClient.createTimelineClient(); timelineClient.init(conf); } else { this.timelineClient = null; if (conf.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "") - .equals(atsHistoryLoggingServiceClassName)) { + .equals(atsHistoryLoggingServiceClassName)) { LOG.warn(atsHistoryLoggingServiceClassName + " is disabled due to Timeline Service being disabled, " + YarnConfiguration.TIMELINE_SERVICE_ENABLED + " set to false"); @@ -189,7 +189,7 @@ public void run() { TezUtilsInternal.setHadoopCallerContext(appContext.getHadoopShim(), appContext.getApplicationID()); while (!stopped.get() && !Thread.currentThread().isInterrupted() - && !interrupted) { + && !interrupted) { // Log the size of the event-queue every so often. if (eventCounter != 0 && eventCounter % 1000 == 0) { @@ -436,11 +436,11 @@ private String getDomainForEvent(DAGHistoryEvent event) { Configuration conf; DAGPlan dagPlan; if (HistoryEventType.DAG_SUBMITTED == historyEvent.getEventType()) { - conf = ((DAGSubmittedEvent)historyEvent).getConf(); - dagPlan = ((DAGSubmittedEvent)historyEvent).getDAGPlan(); + conf = ((DAGSubmittedEvent) historyEvent).getConf(); + dagPlan = ((DAGSubmittedEvent) historyEvent).getDAGPlan(); } else { - conf = appContext.getCurrentDAG().getConf(); - dagPlan = appContext.getCurrentDAG().getJobPlan(); + conf = appContext.getCurrentDAG().getConf(); + dagPlan = appContext.getCurrentDAG().getJobPlan(); } domainId = createDagDomain(conf, dagPlan, dagId); diff --git a/tez-plugins/tez-yarn-timeline-history-with-fs/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryV15.java b/tez-plugins/tez-yarn-timeline-history-with-fs/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryV15.java index f49e588afe..c92a2b2272 100644 --- a/tez-plugins/tez-yarn-timeline-history-with-fs/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryV15.java +++ b/tez-plugins/tez-yarn-timeline-history-with-fs/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryV15.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -114,7 +114,6 @@ public static void setup() throws IOException { conf.set(YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_DONE_DIR, remoteFs.resolvePath(atsDonePath).toString()); - mrrTezCluster.init(conf); mrrTezCluster.start(); } catch (Throwable e) { @@ -143,7 +142,7 @@ public static void tearDown() throws InterruptedException { } } - @Test(timeout=50000) + @Test(timeout = 50000) public void testSimpleDAG() throws Exception { TezClient tezSession = null; ApplicationId applicationId; @@ -303,7 +302,8 @@ public void fromProtoStream(CodedInputStream inputStream) throws IOException { ATSV15HistoryLoggingService service = new ATSV15HistoryLoggingService(); AppContext appContext = mock(AppContext.class); when(appContext.getApplicationID()).thenReturn(appId); - when(appContext.getHadoopShim()).thenReturn(new HadoopShim() {}); + when(appContext.getHadoopShim()).thenReturn(new HadoopShim() { + }); service.setAppContext(appContext); TimelineEntityGroupId grpId = service.getGroupId(event); @@ -323,5 +323,4 @@ public void fromProtoStream(CodedInputStream inputStream) throws IOException { service.close(); } } - } diff --git a/tez-plugins/tez-yarn-timeline-history-with-fs/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSV15HistoryLoggingService.java b/tez-plugins/tez-yarn-timeline-history-with-fs/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSV15HistoryLoggingService.java index 7ba14fffef..e1eb0e9217 100644 --- a/tez-plugins/tez-yarn-timeline-history-with-fs/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSV15HistoryLoggingService.java +++ b/tez-plugins/tez-yarn-timeline-history-with-fs/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSV15HistoryLoggingService.java @@ -78,11 +78,11 @@ public class TestATSV15HistoryLoggingService { private TimelineClient timelineClient; Map> entityLog; final TimelineEntityGroupId DEFAULT_GROUP_ID = - TimelineEntityGroupId.newInstance(ApplicationId.newInstance(0, -1), ""); + TimelineEntityGroupId.newInstance(ApplicationId.newInstance(0, -1), ""); private AppContext appContext; - @Test(timeout=2000) + @Test(timeout = 2000) public void testDAGGroupingDefault() throws Exception { ATSV15HistoryLoggingService service = createService(-1); @@ -111,7 +111,7 @@ public void testDAGGroupingDefault() throws Exception { service.stop(); } - @Test(timeout=2000) + @Test(timeout = 2000) public void testDAGGroupingDisabled() throws Exception { ATSV15HistoryLoggingService service = createService(1); service.start(); @@ -139,7 +139,7 @@ public void testDAGGroupingDisabled() throws Exception { service.stop(); } - @Test(timeout=2000) + @Test(timeout = 2000) public void testDAGGroupingGroupingEnabled() throws Exception { int numDagsPerGroup = 100; ATSV15HistoryLoggingService service = createService(numDagsPerGroup); @@ -149,7 +149,7 @@ public void testDAGGroupingGroupingEnabled() throws Exception { for (DAGHistoryEvent event : makeHistoryEvents(dagId1, service)) { service.handle(event); } - TezDAGID dagId2 = TezDAGID.getInstance(appId, numDagsPerGroup ); + TezDAGID dagId2 = TezDAGID.getInstance(appId, numDagsPerGroup); for (DAGHistoryEvent event : makeHistoryEvents(dagId2, service)) { service.handle(event); } @@ -201,14 +201,14 @@ public void testNonSessionDomains() throws Exception { HistoryACLPolicyManager historyACLPolicyManager = mock(HistoryACLPolicyManager.class); service.historyACLPolicyManager = historyACLPolicyManager; - when(historyACLPolicyManager.setupSessionACLs((Configuration)any(), eq(appId))) - .thenReturn(Collections.singletonMap( - TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID, "session-id")); + when(historyACLPolicyManager.setupSessionACLs((Configuration) any(), eq(appId))) + .thenReturn(Collections.singletonMap( + TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID, "session-id")); service.start(); verify(historyACLPolicyManager, times(1)) - .setupSessionACLs((Configuration)any(), eq(appId)); + .setupSessionACLs((Configuration) any(), eq(appId)); // Send the event and wait for completion. TezDAGID dagId1 = TezDAGID.getInstance(appId, 0); @@ -220,7 +220,7 @@ public void testNonSessionDomains() throws Exception { } // No dag domain were created. verify(historyACLPolicyManager, times(0)) - .setupSessionDAGACLs((Configuration)any(), eq(appId), eq("0"), (DAGAccessControls)any()); + .setupSessionDAGACLs((Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any()); // All calls made with session domain id. // NOTE: Expect 6 invocations for 5 history events because DAG_SUBMITTED becomes two separate timeline events. @@ -237,12 +237,12 @@ public void testNonSessionDomainsFailed() throws Exception { HistoryACLPolicyManager historyACLPolicyManager = mock(HistoryACLPolicyManager.class); service.historyACLPolicyManager = historyACLPolicyManager; - when(historyACLPolicyManager.setupSessionACLs((Configuration)any(), eq(appId))) - .thenThrow(new IOException()); + when(historyACLPolicyManager.setupSessionACLs((Configuration) any(), eq(appId))) + .thenThrow(new IOException()); service.start(); - verify(historyACLPolicyManager, times(1)).setupSessionACLs((Configuration)any(), eq(appId)); + verify(historyACLPolicyManager, times(1)).setupSessionACLs((Configuration) any(), eq(appId)); // Send the event and wait for completion. TezDAGID dagId1 = TezDAGID.getInstance(appId, 0); @@ -254,10 +254,10 @@ public void testNonSessionDomainsFailed() throws Exception { } // No dag domain were created. verify(historyACLPolicyManager, times(0)) - .setupSessionDAGACLs((Configuration)any(), eq(appId), eq("0"), (DAGAccessControls)any()); + .setupSessionDAGACLs((Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any()); // History logging is disabled. - verify(historyACLPolicyManager, times(0)).updateTimelineEntityDomain(any(), (String)any()); + verify(historyACLPolicyManager, times(0)).updateTimelineEntityDomain(any(), (String) any()); assertEquals(0, entityLog.size()); service.stop(); @@ -270,12 +270,12 @@ public void testNonSessionDomainsAclNull() throws Exception { HistoryACLPolicyManager historyACLPolicyManager = mock(HistoryACLPolicyManager.class); service.historyACLPolicyManager = historyACLPolicyManager; - when(historyACLPolicyManager.setupSessionACLs((Configuration)any(), eq(appId))) - .thenReturn(null); + when(historyACLPolicyManager.setupSessionACLs((Configuration) any(), eq(appId))) + .thenReturn(null); service.start(); - verify(historyACLPolicyManager, times(1)).setupSessionACLs((Configuration)any(), eq(appId)); + verify(historyACLPolicyManager, times(1)).setupSessionACLs((Configuration) any(), eq(appId)); // Send the event and wait for completion. TezDAGID dagId1 = TezDAGID.getInstance(appId, 0); @@ -287,10 +287,10 @@ public void testNonSessionDomainsAclNull() throws Exception { } // No dag domain were created. verify(historyACLPolicyManager, times(0)) - .setupSessionDAGACLs((Configuration)any(), eq(appId), eq("0"), (DAGAccessControls)any()); + .setupSessionDAGACLs((Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any()); // No domain updates but history logging happened. - verify(historyACLPolicyManager, times(0)).updateTimelineEntityDomain(any(), (String)any()); + verify(historyACLPolicyManager, times(0)).updateTimelineEntityDomain(any(), (String) any()); assertTrue(entityLog.size() > 0); service.stop(); @@ -305,20 +305,20 @@ public void testSessionDomains() throws Exception { HistoryACLPolicyManager historyACLPolicyManager = mock(HistoryACLPolicyManager.class); service.historyACLPolicyManager = historyACLPolicyManager; - when(historyACLPolicyManager.setupSessionACLs((Configuration)any(), eq(appId))) - .thenReturn(Collections.singletonMap( - TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID, "session-id")); + when(historyACLPolicyManager.setupSessionACLs((Configuration) any(), eq(appId))) + .thenReturn(Collections.singletonMap( + TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID, "session-id")); service.start(); // Verify that the session domain was created. - verify(historyACLPolicyManager, times(1)).setupSessionACLs((Configuration)any(), eq(appId)); + verify(historyACLPolicyManager, times(1)).setupSessionACLs((Configuration) any(), eq(appId)); // Mock dag domain creation. when(historyACLPolicyManager.setupSessionDAGACLs( - (Configuration)any(), eq(appId), eq("0"), (DAGAccessControls)any())) - .thenReturn( - Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_DAG_DOMAIN_ID, "dag-id")); + (Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any())) + .thenReturn( + Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_DAG_DOMAIN_ID, "dag-id")); // Send the event and wait for completion. TezDAGID dagId1 = TezDAGID.getInstance(appId, 0); @@ -330,7 +330,7 @@ public void testSessionDomains() throws Exception { } // Verify dag domain was created. verify(historyACLPolicyManager, times(1)) - .setupSessionDAGACLs((Configuration)any(), eq(appId), eq("0"), (DAGAccessControls)any()); + .setupSessionDAGACLs((Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any()); // calls were made with correct domain ids. verify(historyACLPolicyManager, times(1)).updateTimelineEntityDomain(any(), eq("session-id")); @@ -348,19 +348,19 @@ public void testSessionDomainsFailed() throws Exception { HistoryACLPolicyManager historyACLPolicyManager = mock(HistoryACLPolicyManager.class); service.historyACLPolicyManager = historyACLPolicyManager; - when(historyACLPolicyManager.setupSessionACLs((Configuration)any(), eq(appId))) - .thenThrow(new IOException()); + when(historyACLPolicyManager.setupSessionACLs((Configuration) any(), eq(appId))) + .thenThrow(new IOException()); service.start(); // Verify that the session domain was created. - verify(historyACLPolicyManager, times(1)).setupSessionACLs((Configuration)any(), eq(appId)); + verify(historyACLPolicyManager, times(1)).setupSessionACLs((Configuration) any(), eq(appId)); // Mock dag domain creation. when(historyACLPolicyManager.setupSessionDAGACLs( - (Configuration)any(), eq(appId), eq("0"), (DAGAccessControls)any())) - .thenReturn( - Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_DAG_DOMAIN_ID, "dag-id")); + (Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any())) + .thenReturn( + Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_DAG_DOMAIN_ID, "dag-id")); // Send the event and wait for completion. TezDAGID dagId1 = TezDAGID.getInstance(appId, 0); @@ -372,10 +372,10 @@ public void testSessionDomainsFailed() throws Exception { } // No dag creation was done. verify(historyACLPolicyManager, times(0)) - .setupSessionDAGACLs((Configuration)any(), eq(appId), eq("0"), (DAGAccessControls)any()); + .setupSessionDAGACLs((Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any()); // No history logging calls were done - verify(historyACLPolicyManager, times(0)).updateTimelineEntityDomain(any(), (String)any()); + verify(historyACLPolicyManager, times(0)).updateTimelineEntityDomain(any(), (String) any()); assertEquals(0, entityLog.size()); service.stop(); @@ -390,19 +390,19 @@ public void testSessionDomainsDagFailed() throws Exception { HistoryACLPolicyManager historyACLPolicyManager = mock(HistoryACLPolicyManager.class); service.historyACLPolicyManager = historyACLPolicyManager; - when(historyACLPolicyManager.setupSessionACLs((Configuration)any(), eq(appId))) - .thenReturn( - Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID, "session-id")); + when(historyACLPolicyManager.setupSessionACLs((Configuration) any(), eq(appId))) + .thenReturn( + Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID, "session-id")); service.start(); // Verify that the session domain creation was called. - verify(historyACLPolicyManager, times(1)).setupSessionACLs((Configuration)any(), eq(appId)); + verify(historyACLPolicyManager, times(1)).setupSessionACLs((Configuration) any(), eq(appId)); // Mock dag domain creation. when(historyACLPolicyManager.setupSessionDAGACLs( - (Configuration)any(), eq(appId), eq("0"), (DAGAccessControls)any())) - .thenThrow(new IOException()); + (Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any())) + .thenThrow(new IOException()); // Send the event and wait for completion. TezDAGID dagId1 = TezDAGID.getInstance(appId, 0); @@ -414,7 +414,7 @@ public void testSessionDomainsDagFailed() throws Exception { } // Verify dag domain creation was called. verify(historyACLPolicyManager, times(1)) - .setupSessionDAGACLs((Configuration)any(), eq(appId), eq("0"), (DAGAccessControls)any()); + .setupSessionDAGACLs((Configuration) any(), eq(appId), eq("0"), (DAGAccessControls) any()); // AM events sent, dag events are not sent. verify(historyACLPolicyManager, times(1)).updateTimelineEntityDomain(any(), eq("session-id")); @@ -428,7 +428,8 @@ private ATSV15HistoryLoggingService createService(int numDagsPerGroup) throws IO ATSV15HistoryLoggingService service = new ATSV15HistoryLoggingService(); appContext = mock(AppContext.class); when(appContext.getApplicationID()).thenReturn(appId); - when(appContext.getHadoopShim()).thenReturn(new HadoopShim() {}); + when(appContext.getHadoopShim()).thenReturn(new HadoopShim() { + }); service.setAppContext(appContext); Configuration conf = new Configuration(false); diff --git a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java index c1883a9216..708338cb87 100644 --- a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java +++ b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/ATSHistoryLoggingService.java @@ -105,13 +105,13 @@ public void serviceInit(Configuration conf) throws Exception { } if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { timelineClient = TimelineClient.createTimelineClient(); timelineClient.init(conf); } else { this.timelineClient = null; if (conf.get(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, "") - .equals(atsHistoryLoggingServiceClassName)) { + .equals(atsHistoryLoggingServiceClassName)) { LOG.warn(atsHistoryLoggingServiceClassName + " is disabled due to Timeline Service being disabled, " + YarnConfiguration.TIMELINE_SERVICE_ENABLED + " set to false"); @@ -131,10 +131,10 @@ public void serviceInit(Configuration conf) throws Exception { } LOG.info("Initializing " + ATSHistoryLoggingService.class.getSimpleName() + " with " - + "maxEventsPerBatch=" + maxEventsPerBatch - + ", maxPollingTime(ms)=" + maxPollingTimeMillis - + ", waitTimeForShutdown(ms)=" + maxTimeToWaitOnShutdown - + ", TimelineACLManagerClass=" + atsHistoryACLManagerClassName); + + "maxEventsPerBatch=" + maxEventsPerBatch + + ", maxPollingTime(ms)=" + maxPollingTimeMillis + + ", waitTimeForShutdown(ms)=" + maxTimeToWaitOnShutdown + + ", TimelineACLManagerClass=" + atsHistoryACLManagerClassName); try { historyACLPolicyManager = ReflectionUtils.createClazzInstance( @@ -149,7 +149,6 @@ public void serviceInit(Configuration conf) throws Exception { } historyACLPolicyManager = null; } - } @Override @@ -174,7 +173,7 @@ public void run() { List events = new LinkedList(); boolean interrupted = false; while (!stopped.get() && !Thread.currentThread().isInterrupted() - && !interrupted) { + && !interrupted) { // Log the size of the event-queue every so often. if (eventCounter != 0 && eventCounter % 1000 == 0) { @@ -236,7 +235,7 @@ public void serviceStop() { getEventBatch(events); } catch (InterruptedException e) { LOG.info("ATSService interrupted while shutting down. Exiting." - + " EventQueueBacklog=" + eventQueue.size()); + + " EventQueueBacklog=" + eventQueue.size()); } if (events.isEmpty()) { LOG.info("Event queue empty, stopping ATS Service"); @@ -340,7 +339,7 @@ private void handleEvents(List events) { event.getHistoryEvent()); entities.addAll(eventEntities); if (historyACLPolicyManager != null && domainId != null && !domainId.isEmpty()) { - for (TimelineEntity entity: eventEntities) { + for (TimelineEntity entity : eventEntities) { historyACLPolicyManager.updateTimelineEntityDomain(entity, domainId); } } @@ -353,7 +352,7 @@ private void handleEvents(List events) { TimelinePutResponse response = timelineClient.putEntities(entities.toArray(new TimelineEntity[entities.size()])); if (response != null - && !response.getErrors().isEmpty()) { + && !response.getErrors().isEmpty()) { int count = response.getErrors().size(); for (int i = 0; i < count; ++i) { TimelinePutError err = response.getErrors().get(i); @@ -396,11 +395,11 @@ private String getDomainForEvent(DAGHistoryEvent event) { Configuration conf; DAGPlan dagPlan; if (HistoryEventType.DAG_SUBMITTED == historyEvent.getEventType()) { - conf = ((DAGSubmittedEvent)historyEvent).getConf(); - dagPlan = ((DAGSubmittedEvent)historyEvent).getDAGPlan(); + conf = ((DAGSubmittedEvent) historyEvent).getConf(); + dagPlan = ((DAGSubmittedEvent) historyEvent).getDAGPlan(); } else { - conf = appContext.getCurrentDAG().getConf(); - dagPlan = appContext.getCurrentDAG().getJobPlan(); + conf = appContext.getCurrentDAG().getConf(); + dagPlan = appContext.getCurrentDAG().getJobPlan(); } domainId = createDagDomain(conf, dagPlan, dagId); @@ -416,6 +415,7 @@ private String getDomainForEvent(DAGHistoryEvent event) { /** * Creates a domain for the session. + * * @return domainId to be used. null if acls are disabled. * @throws HistoryACLPolicyException, IOException Forward if historyACLPolicyManger exception. */ @@ -433,12 +433,13 @@ private String createSessionDomain() throws HistoryACLPolicyException, IOExcepti /** * When running in session mode, create a domain for the dag and return it. + * * @param dagConf The configuration the dag for which domain has to be created. * @param dagPlan The dag plan which contains the ACLs. - * @param dagId The dagId for which domain has to be created. + * @param dagId The dagId for which domain has to be created. * @return The created domain id on success. - * sessionDomainId: If there is a failure also disable history logging for this dag. - * sessionDomainId: If historyACLPolicyManager returns null. + * sessionDomainId: If there is a failure also disable history logging for this dag. + * sessionDomainId: If historyACLPolicyManager returns null. */ private String createDagDomain(Configuration dagConf, DAGPlan dagPlan, TezDAGID dagId) { // In non session mode dag domain is same as session domain id. diff --git a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java index b9457a0761..f245f306a4 100644 --- a/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java +++ b/tez-plugins/tez-yarn-timeline-history/src/main/java/org/apache/tez/dag/history/logging/ats/HistoryEventTimelineConversion.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -86,8 +86,8 @@ public static List convertToTimelineEntities(HistoryEvent histor convertContainerStoppedEvent((ContainerStoppedEvent) historyEvent)); case DAG_SUBMITTED: return Lists.newArrayList( - convertDAGSubmittedEvent((DAGSubmittedEvent)historyEvent), - convertDAGSubmittedToDAGExtraInfoEntity((DAGSubmittedEvent)historyEvent)); + convertDAGSubmittedEvent((DAGSubmittedEvent) historyEvent), + convertDAGSubmittedToDAGExtraInfoEntity((DAGSubmittedEvent) historyEvent)); case DAG_INITIALIZED: return Collections.singletonList( convertDAGInitializedEvent((DAGInitializedEvent) historyEvent)); @@ -177,7 +177,7 @@ private static TimelineEntity convertAppLaunchedEvent(AppLaunchedEvent event) { atsEntity.addOtherInfo(ATSConstants.CONFIG, DAGUtils.convertConfigurationToATSMap(event.getConf())); atsEntity.addOtherInfo(ATSConstants.APPLICATION_ID, - event.getApplicationId().toString()); + event.getApplicationId().toString()); atsEntity.addOtherInfo(ATSConstants.USER, event.getUser()); atsEntity.setStartTime(event.getLaunchTime()); @@ -316,7 +316,7 @@ private static TimelineEntity convertDAGFinishedEvent(DAGFinishedEvent event) { final Map dagTaskStats = event.getDagTaskStats(); if (dagTaskStats != null) { - for(Entry entry : dagTaskStats.entrySet()) { + for (Entry entry : dagTaskStats.entrySet()) { atsEntity.addOtherInfo(entry.getKey(), entry.getValue()); } } @@ -341,7 +341,6 @@ private static TimelineEntity convertDAGFinishedToDAGExtraInfoEntity(DAGFinished return atsEntity; } - private static TimelineEntity convertDAGInitializedEvent(DAGInitializedEvent event) { TimelineEntity atsEntity = new TimelineEntity(); atsEntity.setEntityId(event.getDAGID().toString()); @@ -427,7 +426,7 @@ private static TimelineEntity convertDAGSubmittedEvent(DAGSubmittedEvent event) atsEntity.addOtherInfo(ATSConstants.APPLICATION_ID, event.getApplicationAttemptId().getApplicationId().toString()); atsEntity.addOtherInfo(ATSConstants.APPLICATION_ATTEMPT_ID, - event.getApplicationAttemptId().toString()); + event.getApplicationAttemptId().toString()); atsEntity.addOtherInfo(ATSConstants.USER, event.getUser()); atsEntity.addOtherInfo(ATSConstants.DAG_AM_WEB_SERVICE_VERSION, AMWebController.VERSION); atsEntity.addOtherInfo(ATSConstants.IN_PROGRESS_LOGS_URL + "_" @@ -511,7 +510,7 @@ private static TimelineEntity convertTaskAttemptFinishedEvent(TaskAttemptFinishe atsEntity.addOtherInfo(ATSConstants.COUNTERS, DAGUtils.convertCountersToATSMap(event.getCounters())); if (event.getDataEvents() != null && !event.getDataEvents().isEmpty()) { - atsEntity.addOtherInfo(ATSConstants.LAST_DATA_EVENTS, + atsEntity.addOtherInfo(ATSConstants.LAST_DATA_EVENTS, DAGUtils.convertDataEventDependecyInfoToATS(event.getDataEvents())); } if (event.getNodeId() != null) { @@ -670,7 +669,7 @@ private static TimelineEntity convertVertexFinishedEvent(VertexFinishedEvent eve final Map vertexTaskStats = event.getVertexTaskStats(); if (vertexTaskStats != null) { - for(Entry entry : vertexTaskStats.entrySet()) { + for (Entry entry : vertexTaskStats.entrySet()) { atsEntity.addOtherInfo(entry.getKey(), entry.getValue()); } } @@ -748,7 +747,7 @@ private static TimelineEntity convertVertexReconfigureDoneEvent( updateEvt.setEventType(HistoryEventType.VERTEX_CONFIGURE_DONE.name()); updateEvt.setTimestamp(event.getReconfigureDoneTime()); - Map eventInfo = new HashMap<>(); + Map eventInfo = new HashMap<>(); if (event.getSourceEdgeProperties() != null && !event.getSourceEdgeProperties().isEmpty()) { Map updatedEdgeManagers = new HashMap<>(); for (Entry entry : @@ -766,5 +765,4 @@ private static TimelineEntity convertVertexReconfigureDoneEvent( return atsEntity; } - } diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java index 0536568ab4..9de76a6703 100644 --- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java +++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryLoggingService.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -121,7 +121,7 @@ public void teardown() { atsHistoryLoggingService = null; } - @Test(timeout=20000) + @Test(timeout = 20000) public void testATSHistoryLoggingServiceShutdown() { atsHistoryLoggingService.start(); TezDAGID tezDAGID = TezDAGID.getInstance( @@ -145,10 +145,9 @@ public void testATSHistoryLoggingServiceShutdown() { Assert.assertTrue(atsEntitiesCounter >= 4); Assert.assertTrue(atsEntitiesCounter < 20); - } - @Test(timeout=20000) + @Test(timeout = 20000) public void testATSEventBatching() { atsHistoryLoggingService.start(); TezDAGID tezDAGID = TezDAGID.getInstance( @@ -169,10 +168,10 @@ public void testATSEventBatching() { + ", timelineInvocations=" + atsInvokeCounter); Assert.assertTrue(atsEntitiesCounter > atsInvokeCounter); - Assert.assertEquals(atsEntitiesCounter/2, atsInvokeCounter); + Assert.assertEquals(atsEntitiesCounter / 2, atsInvokeCounter); } - @Test(timeout=20000) + @Test(timeout = 20000) public void testTimelineServiceDisable() throws Exception { atsHistoryLoggingService.start(); ATSHistoryLoggingService atsHistoryLoggingService1; @@ -181,46 +180,46 @@ public void testTimelineServiceDisable() throws Exception { atsHistoryLoggingService1.setAppContext(appContext); atsHistoryLoggingService1.timelineClient = mock(TimelineClient.class); when(atsHistoryLoggingService1.timelineClient.putEntities(any())).thenAnswer( - new Answer() { - @Override - public Object answer(InvocationOnMock invocation) throws Throwable { - ++atsInvokeCounter; - atsEntitiesCounter += invocation.getArguments().length; - try { - Thread.sleep(10l); - } catch (InterruptedException e) { - // do nothing - } - return null; - } - }); + new Answer() { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + ++atsInvokeCounter; + atsEntitiesCounter += invocation.getArguments().length; + try { + Thread.sleep(10l); + } catch (InterruptedException e) { + // do nothing + } + return null; + } + }); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false); conf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS, - ATSHistoryLoggingService.class.getName()); + ATSHistoryLoggingService.class.getName()); atsHistoryLoggingService1.init(conf); atsHistoryLoggingService1.start(); TezDAGID tezDAGID = TezDAGID.getInstance( - ApplicationId.newInstance(100l, 1), 1); + ApplicationId.newInstance(100l, 1), 1); DAGHistoryEvent historyEvent = new DAGHistoryEvent(tezDAGID, - new DAGStartedEvent(tezDAGID, 1001l, "user1", "dagName1")); + new DAGStartedEvent(tezDAGID, 1001l, "user1", "dagName1")); for (int i = 0; i < 100; ++i) { atsHistoryLoggingService1.handle(historyEvent); } try { - Thread.sleep(20l); + Thread.sleep(20l); } catch (InterruptedException e) { - // Do nothing + // Do nothing } LOG.info("ATS entitiesSent=" + atsEntitiesCounter - + ", timelineInvocations=" + atsInvokeCounter); + + ", timelineInvocations=" + atsInvokeCounter); Assert.assertEquals(atsInvokeCounter, 0); Assert.assertEquals(atsEntitiesCounter, 0); Assert.assertNull(atsHistoryLoggingService1.timelineClient); atsHistoryLoggingService1.close(); } - @Test(timeout=10000) + @Test(timeout = 10000) public void testNonSessionDomains() throws Exception { when(historyACLPolicyManager.setupSessionACLs(any(), any())) .thenReturn( @@ -239,13 +238,13 @@ public void testNonSessionDomains() throws Exception { } // No dag domain were created. verify(historyACLPolicyManager, times(0)) - .setupSessionDAGACLs(any(), eq(appId), eq("0"), any()); + .setupSessionDAGACLs(any(), eq(appId), eq("0"), any()); // All calls made with session domain id. verify(historyACLPolicyManager, times(6)).updateTimelineEntityDomain(any(), eq("session-id")); } - @Test(timeout=10000) + @Test(timeout = 10000) public void testNonSessionDomainsFailed() throws Exception { when(historyACLPolicyManager.setupSessionACLs(any(), any())) .thenThrow(new IOException()); @@ -269,7 +268,7 @@ public void testNonSessionDomainsFailed() throws Exception { Assert.assertEquals(0, atsEntitiesCounter); } - @Test(timeout=10000) + @Test(timeout = 10000) public void testNonSessionDomainsAclNull() throws Exception { when(historyACLPolicyManager.setupSessionACLs(any(), any())) .thenReturn(null); @@ -294,7 +293,7 @@ public void testNonSessionDomainsAclNull() throws Exception { Assert.assertEquals(6, atsEntitiesCounter); } - @Test(timeout=10000) + @Test(timeout = 10000) public void testSessionDomains() throws Exception { when(historyACLPolicyManager.setupSessionACLs(any(), any())) .thenReturn(Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID, "test-domain")); @@ -324,7 +323,7 @@ public void testSessionDomains() throws Exception { verify(historyACLPolicyManager, times(5)).updateTimelineEntityDomain(any(), eq("dag-domain")); } - @Test(timeout=10000) + @Test(timeout = 10000) public void testSessionDomainsFailed() throws Exception { when(historyACLPolicyManager.setupSessionACLs(any(), any())) .thenThrow(new IOException()); @@ -353,7 +352,7 @@ public void testSessionDomainsFailed() throws Exception { Assert.assertEquals(0, atsEntitiesCounter); } - @Test(timeout=10000) + @Test(timeout = 10000) public void testSessionDomainsDagFailed() throws Exception { when(historyACLPolicyManager.setupSessionACLs(any(), any())) .thenReturn(Collections.singletonMap(TezConfiguration.YARN_ATS_ACL_SESSION_DOMAIN_ID, "session-domain")); @@ -386,7 +385,7 @@ public void testSessionDomainsDagFailed() throws Exception { Assert.assertEquals(1, atsEntitiesCounter); } - @Test(timeout=10000) + @Test(timeout = 10000) public void testSessionDomainsAclNull() throws Exception { when(historyACLPolicyManager.setupSessionACLs(any(), any())) .thenReturn(null); @@ -417,7 +416,7 @@ public void testSessionDomainsAclNull() throws Exception { } private List makeHistoryEvents(TezDAGID dagId, - ATSHistoryLoggingService service) { + ATSHistoryLoggingService service) { List historyEvents = new ArrayList<>(); long time = System.currentTimeMillis(); diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryWithMiniCluster.java b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryWithMiniCluster.java index 448de71416..24d07226dd 100644 --- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryWithMiniCluster.java +++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryWithMiniCluster.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -130,7 +130,7 @@ private K getTimelineData(String url, Class clazz) { return entity; } - @Test (timeout=50000) + @Test(timeout = 50000) public void testDisabledACls() throws Exception { TezClient tezSession = null; try { @@ -170,5 +170,4 @@ public void testDisabledACls() throws Exception { } } } - } diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java index 1663cb0227..24d2a04915 100644 --- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java +++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestHistoryEventTimelineConversion.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -230,7 +230,6 @@ static class MockVersionInfo extends VersionInfo { MockVersionInfo() { super("component", "1.1.0", "rev1", "20120101", "git.apache.org"); } - } @Test(timeout = 5000) @@ -315,7 +314,6 @@ public void testConvertAppLaunchedEvent() { versionInfo.get(ATSConstants.REVISION)); Assert.assertEquals(mockVersionInfo.getBuildTime(), versionInfo.get(ATSConstants.BUILD_TIME)); - } @Test(timeout = 5000) @@ -375,13 +373,12 @@ public void testConvertAMStartedEvent() { Assert.assertEquals(2, primaryFilters.size()); Assert.assertTrue(primaryFilters.get(ATSConstants.USER).contains(user)); Assert.assertTrue(primaryFilters.get(ATSConstants.APPLICATION_ID) - .contains(applicationId.toString())); + .contains(applicationId.toString())); Assert.assertEquals(1, timelineEntity.getEvents().size()); TimelineEvent evt = timelineEntity.getEvents().get(0); Assert.assertEquals(HistoryEventType.AM_STARTED.name(), evt.getEventType()); Assert.assertEquals(startTime, evt.getTimestamp()); - } @Test(timeout = 5000) @@ -460,7 +457,6 @@ public void testConvertDAGStartedEvent() { Assert.assertEquals(1, entities.size()); TimelineEntity timelineEntity = entities.get(0); - Assert.assertEquals(tezDAGID.toString(), timelineEntity.getEntityId()); Assert.assertEquals(EntityTypes.TEZ_DAG_ID.name(), timelineEntity.getEntityType()); @@ -493,7 +489,6 @@ public void testConvertDAGSubmittedEvent() { List entities = HistoryEventTimelineConversion.convertToTimelineEntities(event); Assert.assertEquals(2, entities.size()); - if (entities.get(0).getEntityType().equals(EntityTypes.TEZ_DAG_ID.name())) { assertDagSubmittedEntity(submitTime, event, entities.get(0)); assertDagSubmittedExtraInfoEntity(submitTime, event, entities.get(1)); @@ -504,7 +499,7 @@ public void testConvertDAGSubmittedEvent() { } private void assertDagSubmittedEntity(long submitTime, DAGSubmittedEvent event, - TimelineEntity timelineEntity) { + TimelineEntity timelineEntity) { Assert.assertEquals(EntityTypes.TEZ_DAG_ID.name(), timelineEntity.getEntityType()); Assert.assertEquals(tezDAGID.toString(), timelineEntity.getEntityId()); @@ -556,19 +551,18 @@ private void assertDagSubmittedEntity(long submitTime, DAGSubmittedEvent event, + applicationAttemptId.getAttemptId())); Assert.assertEquals( timelineEntity.getOtherInfo().get(ATSConstants.CALLER_CONTEXT_ID), - dagPlan.getCallerContext().getCallerId()); + dagPlan.getCallerContext().getCallerId()); Assert.assertEquals( timelineEntity.getOtherInfo().get(ATSConstants.CALLER_CONTEXT_TYPE), - dagPlan.getCallerContext().getCallerType()); + dagPlan.getCallerContext().getCallerType()); Assert.assertEquals(dagPlan.getCallerContext().getContext(), timelineEntity.getOtherInfo().get(ATSConstants.CALLER_CONTEXT)); Assert.assertEquals( event.getQueueName(), timelineEntity.getOtherInfo().get(ATSConstants.DAG_QUEUE_NAME)); - } private void assertDagSubmittedExtraInfoEntity(long submitTime, DAGSubmittedEvent event, - TimelineEntity timelineEntity) { + TimelineEntity timelineEntity) { Assert.assertEquals(EntityTypes.TEZ_DAG_EXTRA_INFO.name(), timelineEntity.getEntityType()); Assert.assertEquals(tezDAGID.toString(), timelineEntity.getEntityId()); @@ -635,7 +629,7 @@ public void testConvertTaskAttemptFinishedEvent() { final Map otherInfo = timelineEntity.getOtherInfo(); Assert.assertEquals(17, otherInfo.size()); - Assert.assertEquals(tezTaskAttemptID.toString(), + Assert.assertEquals(tezTaskAttemptID.toString(), timelineEntity.getOtherInfo().get(ATSConstants.CREATION_CAUSAL_ATTEMPT)); Assert.assertEquals(creationTime, timelineEntity.getOtherInfo().get(ATSConstants.CREATION_TIME)); Assert.assertEquals(allocationTime, timelineEntity.getOtherInfo().get(ATSConstants.ALLOCATION_TIME)); @@ -646,7 +640,7 @@ public void testConvertTaskAttemptFinishedEvent() { Assert.assertEquals(TaskFailureType.FATAL.name(), otherInfo.get(ATSConstants.TASK_FAILURE_TYPE)); Assert.assertEquals(error.name(), otherInfo.get(ATSConstants.TASK_ATTEMPT_ERROR_ENUM)); Assert.assertEquals(diagnostics, otherInfo.get(ATSConstants.DIAGNOSTICS)); - Map obj1 = (Map)otherInfo.get(ATSConstants.LAST_DATA_EVENTS); + Map obj1 = (Map) otherInfo.get(ATSConstants.LAST_DATA_EVENTS); List obj2 = (List) obj1.get(ATSConstants.LAST_DATA_EVENTS); Assert.assertEquals(2, obj2.size()); Map obj3 = (Map) obj2.get(0); @@ -684,7 +678,6 @@ public void testConvertDAGInitializedEvent() { DAGInitializedEvent event = new DAGInitializedEvent(tezDAGID, initTime, "user", "dagName", nameIdMap); - List entities = HistoryEventTimelineConversion.convertToTimelineEntities(event); Assert.assertEquals(1, entities.size()); TimelineEntity timelineEntity = entities.get(0); @@ -715,7 +708,6 @@ public void testConvertDAGInitializedEvent() { Assert.assertEquals(1, vIdMap.size()); Assert.assertNotNull(vIdMap.containsKey("foo")); Assert.assertEquals(tezVertexID.toString(), vIdMap.get("foo")); - } @Test(timeout = 5000) @@ -742,7 +734,7 @@ public void testConvertDAGFinishedEvent() { } private void assertDagFinishedEntity(long finishTime, long startTime, DAGFinishedEvent event, - TimelineEntity timelineEntity) { + TimelineEntity timelineEntity) { Assert.assertEquals(EntityTypes.TEZ_DAG_ID.name(), timelineEntity.getEntityType()); Assert.assertEquals(tezDAGID.toString(), timelineEntity.getEntityId()); @@ -818,7 +810,6 @@ public void testConvertVertexInitializedEvent() { .setTaskSchedulerClassName("def1") .setTaskCommunicatorClassName("ghi1")); - List entities = HistoryEventTimelineConversion.convertToTimelineEntities(event); Assert.assertEquals(1, entities.size()); TimelineEntity timelineEntity = entities.get(0); @@ -859,22 +850,22 @@ public void testConvertVertexInitializedEvent() { ((Integer) timelineEntity.getOtherInfo().get(ATSConstants.NUM_TASKS)).intValue()); Assert.assertNotNull(timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)); Assert.assertEquals("abc", - ((Map)timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)).get( + ((Map) timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)).get( ATSConstants.CONTAINER_LAUNCHER_NAME)); Assert.assertEquals("def", - ((Map)timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)).get( + ((Map) timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)).get( ATSConstants.TASK_SCHEDULER_NAME)); Assert.assertEquals("ghi", - ((Map)timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)).get( + ((Map) timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)).get( ATSConstants.TASK_COMMUNICATOR_NAME)); Assert.assertEquals("abc1", - ((Map)timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)).get( + ((Map) timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)).get( ATSConstants.CONTAINER_LAUNCHER_CLASS_NAME)); Assert.assertEquals("def1", - ((Map)timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)).get( + ((Map) timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)).get( ATSConstants.TASK_SCHEDULER_CLASS_NAME)); Assert.assertEquals("ghi1", - ((Map)timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)).get( + ((Map) timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)).get( ATSConstants.TASK_COMMUNICATOR_CLASS_NAME)); } @@ -896,10 +887,10 @@ public void testConvertVertexStartedEvent() { Assert.assertEquals(2, timelineEntity.getPrimaryFilters().size()); Assert.assertTrue(timelineEntity.getPrimaryFilters().get(ATSConstants.APPLICATION_ID) - .contains(applicationId.toString())); + .contains(applicationId.toString())); Assert.assertTrue( - timelineEntity.getPrimaryFilters().get(EntityTypes.TEZ_DAG_ID.name()).contains( - tezDAGID.toString())); + timelineEntity.getPrimaryFilters().get(EntityTypes.TEZ_DAG_ID.name()).contains( + tezDAGID.toString())); Assert.assertEquals(1, timelineEntity.getEvents().size()); TimelineEvent timelineEvent = timelineEntity.getEvents().get(0); @@ -908,11 +899,11 @@ public void testConvertVertexStartedEvent() { Assert.assertEquals(3, timelineEntity.getOtherInfo().size()); Assert.assertEquals(startRequestedTime, - ((Long) timelineEntity.getOtherInfo().get(ATSConstants.START_REQUESTED_TIME)).longValue()); + ((Long) timelineEntity.getOtherInfo().get(ATSConstants.START_REQUESTED_TIME)).longValue()); Assert.assertEquals(startTime, - ((Long) timelineEntity.getOtherInfo().get(ATSConstants.START_TIME)).longValue()); + ((Long) timelineEntity.getOtherInfo().get(ATSConstants.START_TIME)).longValue()); Assert.assertEquals(VertexState.RUNNING.name(), - timelineEntity.getOtherInfo().get(ATSConstants.STATUS)); + timelineEntity.getOtherInfo().get(ATSConstants.STATUS)); } @SuppressWarnings("unchecked") @@ -974,22 +965,22 @@ public void testConvertVertexFinishedEvent() { timelineEntity.getOtherInfo().get(ATSConstants.DIAGNOSTICS)); Assert.assertNotNull(timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)); Assert.assertEquals("abc", - ((Map)timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)).get( + ((Map) timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)).get( ATSConstants.CONTAINER_LAUNCHER_NAME)); Assert.assertEquals("def", - ((Map)timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)).get( + ((Map) timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)).get( ATSConstants.TASK_SCHEDULER_NAME)); Assert.assertEquals("ghi", - ((Map)timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)).get( + ((Map) timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)).get( ATSConstants.TASK_COMMUNICATOR_NAME)); Assert.assertEquals("abc1", - ((Map)timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)).get( + ((Map) timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)).get( ATSConstants.CONTAINER_LAUNCHER_CLASS_NAME)); Assert.assertEquals("def1", - ((Map)timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)).get( + ((Map) timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)).get( ATSConstants.TASK_SCHEDULER_CLASS_NAME)); Assert.assertEquals("ghi1", - ((Map)timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)).get( + ((Map) timelineEntity.getOtherInfo().get(ATSConstants.SERVICE_PLUGIN)).get( ATSConstants.TASK_COMMUNICATOR_CLASS_NAME)); Assert.assertTrue(timelineEntity.getOtherInfo().containsKey(ATSConstants.STATS)); @@ -1152,9 +1143,9 @@ public void testConvertVertexReconfigreDoneEvent() { TezVertexID vId = tezVertexID; Map edgeMgrs = new HashMap(); - + edgeMgrs.put("a", EdgeProperty.create(EdgeManagerPluginDescriptor.create("a.class") - .setHistoryText("text"), DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, + .setHistoryText("text"), DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"))); VertexConfigurationDoneEvent event = new VertexConfigurationDoneEvent(vId, 0L, 1, null, edgeMgrs, null, true); @@ -1238,7 +1229,6 @@ public void testConvertDAGRecoveredEvent2() { DAGRecoveredEvent event = new DAGRecoveredEvent(applicationAttemptId, tezDAGID, dagPlan.getName(), user, recoverTime, DAGState.ERROR, "mock reason", containerLogs); - List entities = HistoryEventTimelineConversion.convertToTimelineEntities(event); Assert.assertEquals(1, entities.size()); TimelineEntity timelineEntity = entities.get(0); @@ -1273,6 +1263,4 @@ public void testConvertDAGRecoveredEvent2() { timelineEntity.getOtherInfo().get(ATSConstants.IN_PROGRESS_LOGS_URL + "_" + applicationAttemptId.getAttemptId())); } - - } diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/tests/MiniTezClusterWithTimeline.java b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/tests/MiniTezClusterWithTimeline.java index f8a35a29c3..f62909a289 100644 --- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/tests/MiniTezClusterWithTimeline.java +++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/tests/MiniTezClusterWithTimeline.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.tests; @@ -84,12 +84,12 @@ public MiniTezClusterWithTimeline(String testName, int noOfNMs) { } public MiniTezClusterWithTimeline(String testName, int noOfNMs, - int numLocalDirs, int numLogDirs) { + int numLocalDirs, int numLogDirs) { super(testName, noOfNMs, numLocalDirs, numLogDirs); } public MiniTezClusterWithTimeline(String testName, int noOfNMs, - int numLocalDirs, int numLogDirs, boolean enableAHS) { + int numLocalDirs, int numLogDirs, boolean enableAHS) { super(testName, 1, noOfNMs, numLocalDirs, numLogDirs, enableAHS); } @@ -110,7 +110,7 @@ public void serviceInit(Configuration conf) throws Exception { conf.set(MRJobConfig.MR_AM_STAGING_DIR, new File(getTestWorkDir(), "apps_staging_dir" + Path.SEPARATOR).getAbsolutePath()); } - + if (conf.get(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC) == null) { // nothing defined. set quick delete value conf.setLong(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 0l); @@ -130,7 +130,7 @@ public void serviceInit(Configuration conf) throws Exception { } else { LOG.info("Using Tez AppJar: " + appJarLocalFile.getAbsolutePath()); } - + FileSystem fs = FileSystem.get(conf); Path testRootDir = fs.makeQualified(new Path("target", getName() + "-tmpDir")); Path appRemoteJar = new Path(testRootDir, "TezAppJar.jar"); @@ -146,7 +146,7 @@ public void serviceInit(Configuration conf) throws Exception { conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false); conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false); - conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000"); + conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000"); try { Path stagingPath = FileContext.getFileContext(conf).makeQualified( @@ -166,7 +166,7 @@ public void serviceInit(Configuration conf) throws Exception { .getAbsolutePath()); } } - FileContext fc=FileContext.getFileContext(stagingPath.toUri(), conf); + FileContext fc = FileContext.getFileContext(stagingPath.toUri(), conf); if (fc.util().exists(stagingPath)) { LOG.info(stagingPath + " exists! deleting..."); fc.delete(stagingPath, true); @@ -186,9 +186,9 @@ public void serviceInit(Configuration conf) throws Exception { //configure the shuffle service in NM conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, - new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID }); + new String[]{ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID}); conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, - ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID), ShuffleHandler.class, + ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID), ShuffleHandler.class, Service.class); // Non-standard shuffle port @@ -250,7 +250,7 @@ private void waitForAppsToFinish() { @Override public boolean apply(ApplicationReport appReport) { return EnumSet.of(YarnApplicationState.NEW, YarnApplicationState.NEW_SAVING, - YarnApplicationState.SUBMITTED, YarnApplicationState.ACCEPTED, YarnApplicationState.RUNNING) + YarnApplicationState.SUBMITTED, YarnApplicationState.ACCEPTED, YarnApplicationState.RUNNING) .contains(appReport.getYarnApplicationState()); } }); @@ -300,7 +300,6 @@ public boolean apply(ApplicationReport appReport) { break; } } while (waitEndTime != -1 && waitEndTime > System.currentTimeMillis()); - } } catch (Exception e) { e.printStackTrace(); @@ -308,9 +307,8 @@ public boolean apply(ApplicationReport appReport) { yarnClient.stop(); } } - + public Path getConfigFilePath() { return confFilePath; } - } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/common/ContainerContext.java b/tez-runtime-internals/src/main/java/org/apache/tez/common/ContainerContext.java index 2cb7b32215..12dc447bd3 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/common/ContainerContext.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/common/ContainerContext.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/common/ContainerTask.java b/tez-runtime-internals/src/main/java/org/apache/tez/common/ContainerTask.java index c0a2574f64..445990ef4b 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/common/ContainerTask.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/common/ContainerTask.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -41,7 +41,7 @@ public ContainerTask() { } public ContainerTask(TaskSpec taskSpec, boolean shouldDie, - Map additionalResources, Credentials credentials, boolean credentialsChanged) { + Map additionalResources, Credentials credentials, boolean credentialsChanged) { this.taskSpec = taskSpec; this.shouldDie = shouldDie; this.additionalResources = additionalResources; @@ -56,15 +56,15 @@ public TaskSpec getTaskSpec() { public boolean shouldDie() { return shouldDie; } - + public Map getAdditionalResources() { return this.additionalResources; } - + public Credentials getCredentials() { return this.credentials; } - + public boolean haveCredentialsChanged() { return this.credentialsChanged; } @@ -107,7 +107,7 @@ public void readFields(DataInput in) throws IOException { int numAdditionalResources = in.readInt(); additionalResources = Maps.newHashMap(); if (numAdditionalResources != -1) { - for (int i = 0 ; i < numAdditionalResources ; i++) { + for (int i = 0; i < numAdditionalResources; i++) { String resourceName = in.readUTF(); TezLocalResource localResource = new TezLocalResource(); localResource.readFields(in); @@ -146,9 +146,8 @@ public String toString() { sb.append("none"); } else { sb.append("#tokens=").append(credentials.numberOfTokens()) - .append(", #secretKeys: ").append(credentials.numberOfSecretKeys()); + .append(", #secretKeys: ").append(credentials.numberOfSecretKeys()); } return sb.toString(); } - } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/common/ProtoConverters.java b/tez-runtime-internals/src/main/java/org/apache/tez/common/ProtoConverters.java index e4285708a5..f4bd589643 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/common/ProtoConverters.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/common/ProtoConverters.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -33,7 +33,7 @@ public class ProtoConverters { public static EventProtos.CustomProcessorEventProto convertCustomProcessorEventToProto( - CustomProcessorEvent event) { + CustomProcessorEvent event) { EventProtos.CustomProcessorEventProto.Builder builder = EventProtos.CustomProcessorEventProto.newBuilder(); if (event.getPayload() != null) { @@ -44,9 +44,9 @@ public static EventProtos.CustomProcessorEventProto convertCustomProcessorEventT } public static CustomProcessorEvent convertCustomProcessorEventFromProto( - EventProtos.CustomProcessorEventProto proto) { + EventProtos.CustomProcessorEventProto proto) { return CustomProcessorEvent.create(proto.getUserPayload() != null ? - proto.getUserPayload().asReadOnlyByteBuffer() : null, proto.getVersion()); + proto.getUserPayload().asReadOnlyByteBuffer() : null, proto.getVersion()); } public static EventProtos.DataMovementEventProto convertDataMovementEventToProto( @@ -69,6 +69,7 @@ public static DataMovementEvent convertDataMovementEventFromProto( proto.getUserPayload() != null ? proto.getUserPayload().asReadOnlyByteBuffer() : null); } + public static EventProtos.CompositeRoutedDataMovementEventProto convertCompositeRoutedDataMovementEventToProto( CompositeRoutedDataMovementEvent event) { EventProtos.CompositeRoutedDataMovementEventProto.Builder builder = @@ -109,7 +110,7 @@ public static CompositeDataMovementEvent convertCompositeDataMovementEventFromPr proto.getCount(), proto.hasUserPayload() ? proto.getUserPayload().asReadOnlyByteBuffer() : null); } - + public static EventProtos.VertexManagerEventProto convertVertexManagerEventToProto( VertexManagerEvent event) { EventProtos.VertexManagerEventProto.Builder vmBuilder = VertexManagerEventProto.newBuilder(); @@ -119,7 +120,7 @@ public static EventProtos.VertexManagerEventProto convertVertexManagerEventToPro } return vmBuilder.build(); } - + public static VertexManagerEvent convertVertexManagerEventFromProto( EventProtos.VertexManagerEventProto vmProto) { return VertexManagerEvent.create(vmProto.getTargetVertexName(), @@ -127,7 +128,7 @@ public static VertexManagerEvent convertVertexManagerEventFromProto( } public static EventProtos.RootInputDataInformationEventProto - convertRootInputDataInformationEventToProto(InputDataInformationEvent event) { + convertRootInputDataInformationEventToProto(InputDataInformationEvent event) { EventProtos.RootInputDataInformationEventProto.Builder builder = EventProtos.RootInputDataInformationEventProto.newBuilder(); builder.setSourceIndex(event.getSourceIndex()); @@ -139,7 +140,7 @@ public static VertexManagerEvent convertVertexManagerEventFromProto( } public static InputDataInformationEvent - convertRootInputDataInformationEventFromProto( + convertRootInputDataInformationEventFromProto( EventProtos.RootInputDataInformationEventProto proto) { InputDataInformationEvent diEvent = InputDataInformationEvent.createWithSerializedPayload( proto.getSourceIndex(), @@ -167,5 +168,4 @@ public static InputInitializerEvent convertRootInputInitializerEventFromProto( (proto.hasUserPayload() ? proto.getUserPayload().asReadOnlyByteBuffer() : null)); return event; } - } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/common/TezConverterUtils.java b/tez-runtime-internals/src/main/java/org/apache/tez/common/TezConverterUtils.java index 02dc69c3e5..41ca5f5117 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/common/TezConverterUtils.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/common/TezConverterUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -124,5 +124,4 @@ public static TaskFailureTypeProto failureTypeToProto(TaskFailureType taskFailur // in.readUTF()); // return url; // } - } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/common/TezLocalResource.java b/tez-runtime-internals/src/main/java/org/apache/tez/common/TezLocalResource.java index 3a8358bdca..a85d246a1b 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/common/TezLocalResource.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/common/TezLocalResource.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -33,7 +33,7 @@ public class TezLocalResource implements Writable { public TezLocalResource() { } - + public TezLocalResource(URI uri, long size, long timestamp) { this.uri = uri; this.size = size; diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/common/TezTaskUmbilicalProtocol.java b/tez-runtime-internals/src/main/java/org/apache/tez/common/TezTaskUmbilicalProtocol.java index e3bd589b44..1ca8f90fc8 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/common/TezTaskUmbilicalProtocol.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/common/TezTaskUmbilicalProtocol.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -52,5 +52,4 @@ public interface TezTaskUmbilicalProtocol extends VersionedProtocol { public TezHeartbeatResponse heartbeat(TezHeartbeatRequest request) throws IOException, TezException; - } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/InputReadyTracker.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/InputReadyTracker.java index b2b2b58e15..adcbd43ed4 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/InputReadyTracker.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/InputReadyTracker.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -40,9 +40,9 @@ public class InputReadyTracker { private final ConcurrentMap readyInputs; - + private ConcurrentMap> inputToGroupMap; - + private final ReentrantLock lock = new ReentrantLock(); private final Condition condition = lock.newCondition(); @@ -66,7 +66,6 @@ public void setInputIsReady(Input input) { } } - private void informGroupedInputs(Input input) { if (inputToGroupMap != null) { List mergedInputList = inputToGroupMap.get(input); diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java index 583cc0099a..14e055c5df 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/LogicalIOProcessorRuntimeTask.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -137,7 +137,7 @@ public class LogicalIOProcessorRuntimeTask extends RuntimeTask { /** Maps which will be provided to the processor run method */ final LinkedHashMap runInputMap; final LinkedHashMap runOutputMap; - + private final Map serviceConsumerMetadata; private final Map envMap; @@ -152,7 +152,7 @@ public class LogicalIOProcessorRuntimeTask extends RuntimeTask { private final int appAttemptNumber; private volatile InputReadyTracker inputReadyTracker; - + private volatile ObjectRegistry objectRegistry; private final ExecutionContext ExecutionContext; private final long memAvailable; @@ -166,12 +166,12 @@ public class LogicalIOProcessorRuntimeTask extends RuntimeTask { private Long initStartTimeNs = null; public LogicalIOProcessorRuntimeTask(TaskSpec taskSpec, int appAttemptNumber, - Configuration tezConf, String[] localDirs, TezUmbilical tezUmbilical, - Map serviceConsumerMetadata, Map envMap, - Multimap startedInputsMap, ObjectRegistry objectRegistry, - String pid, ExecutionContext ExecutionContext, long memAvailable, - boolean updateSysCounters, HadoopShim hadoopShim, - TezExecutors sharedExecutor) throws IOException { + Configuration tezConf, String[] localDirs, TezUmbilical tezUmbilical, + Map serviceConsumerMetadata, Map envMap, + Multimap startedInputsMap, ObjectRegistry objectRegistry, + String pid, ExecutionContext ExecutionContext, long memAvailable, + boolean updateSysCounters, HadoopShim hadoopShim, + TezExecutors sharedExecutor) throws IOException { // Note: If adding any fields here, make sure they're cleaned up in the cleanupContext method. // TODO Remove jobToken from here post TEZ-421 super(taskSpec, tezConf, tezUmbilical, pid, updateSysCounters); @@ -266,7 +266,7 @@ public void initialize() throws Exception { } int completedTasks = 0; while (completedTasks < numTasks) { - LOG.info("Waiting for " + (numTasks-completedTasks) + " initializers to finish"); + LOG.info("Waiting for " + (numTasks - completedTasks) + " initializers to finish"); Future future = initializerCompletionService.take(); try { future.get(); @@ -291,10 +291,10 @@ public void initialize() throws Exception { Set groupInputs = Sets.newHashSet(); // Construct Inputs/Outputs map argument for processor.run() // first add the group inputs - if (groupInputSpecs !=null && !groupInputSpecs.isEmpty()) { + if (groupInputSpecs != null && !groupInputSpecs.isEmpty()) { for (GroupInputSpec groupInputSpec : groupInputSpecs) { runInputMap.put(groupInputSpec.getGroupName(), - groupInputsMap.get(groupInputSpec.getGroupName())); + groupInputsMap.get(groupInputSpec.getGroupName())); groupInputs.addAll(groupInputSpec.getGroupVertices()); } } @@ -352,8 +352,6 @@ public void initialize() throws Exception { } LOG.info("AutoStartComplete"); - - // then add the non-grouped inputs for (InputSpec inputSpec : inputSpecs) { if (!groupInputs.contains(inputSpec.getSourceVertexName())) { @@ -387,13 +385,12 @@ public void close() throws Exception { "Can only run while in RUNNING state. Current: " + this.state); this.state.set(State.CLOSED); - List> allCloseInputEvents = Lists.newArrayList(); // Close the Inputs. for (InputSpec inputSpec : inputSpecs) { String srcVertexName = inputSpec.getSourceVertexName(); initializedInputs.remove(srcVertexName); - List closeInputEvents = ((InputFrameworkInterface)inputsMap.get(srcVertexName)).close(); + List closeInputEvents = ((InputFrameworkInterface) inputsMap.get(srcVertexName)).close(); allCloseInputEvents.add(closeInputEvents); } @@ -402,7 +399,7 @@ public void close() throws Exception { for (OutputSpec outputSpec : outputSpecs) { String destVertexName = outputSpec.getDestinationVertexName(); initializedOutputs.remove(destVertexName); - List closeOutputEvents = ((LogicalOutputFrameworkInterface)outputsMap.get(destVertexName)).close(); + List closeOutputEvents = ((LogicalOutputFrameworkInterface) outputsMap.get(destVertexName)).close(); allCloseOutputEvents.add(closeOutputEvents); } @@ -423,7 +420,6 @@ public void close() throws Exception { EventProducerConsumerType.OUTPUT, taskSpec.getVertexName(), destVertexName, taskSpec.getTaskAttemptID()); } - } finally { setTaskDone(); // Clear the interrupt status since the task execution is done. @@ -475,8 +471,7 @@ protected Void _callInternal() throws Exception { inputsMap.put(edgeName, input); inputContextMap.put(edgeName, inputContext); - - List events = ((InputFrameworkInterface)input).initialize(); + List events = ((InputFrameworkInterface) input).initialize(); sendTaskGeneratedEvents(events, EventProducerConsumerType.INPUT, inputContext.getTaskVertexName(), inputContext.getSourceVertexName(), taskSpec.getTaskAttemptID()); @@ -546,7 +541,7 @@ protected Void _callInternal() throws Exception { outputsMap.put(edgeName, output); outputContextMap.put(edgeName, outputContext); - List events = ((OutputFrameworkInterface)output).initialize(); + List events = ((OutputFrameworkInterface) output).initialize(); sendTaskGeneratedEvents(events, EventProducerConsumerType.OUTPUT, outputContext.getTaskVertexName(), outputContext.getDestinationVertexName(), taskSpec.getTaskAttemptID()); @@ -567,20 +562,20 @@ private boolean inputAlreadyStarted(String vertexName, String edgeVertexName) { private void initializeGroupInputs() throws TezException { if (groupInputSpecs != null && !groupInputSpecs.isEmpty()) { - groupInputsMap = new ConcurrentHashMap(groupInputSpecs.size()); - for (GroupInputSpec groupInputSpec : groupInputSpecs) { - LOG.debug("Initializing GroupInput using GroupInputSpec: {}", groupInputSpec); - MergedInputContext mergedInputContext = - new TezMergedInputContextImpl(groupInputSpec.getMergedInputDescriptor().getUserPayload(), - groupInputSpec.getGroupName(), groupInputsMap, inputReadyTracker, localDirs, this); - List inputs = Lists.newArrayListWithCapacity(groupInputSpec.getGroupVertices().size()); - for (String groupVertex : groupInputSpec.getGroupVertices()) { - inputs.add(inputsMap.get(groupVertex)); - } - - MergedLogicalInput groupInput = - (MergedLogicalInput) createMergedInput(groupInputSpec.getMergedInputDescriptor(), - mergedInputContext, inputs); + groupInputsMap = new ConcurrentHashMap(groupInputSpecs.size()); + for (GroupInputSpec groupInputSpec : groupInputSpecs) { + LOG.debug("Initializing GroupInput using GroupInputSpec: {}", groupInputSpec); + MergedInputContext mergedInputContext = + new TezMergedInputContextImpl(groupInputSpec.getMergedInputDescriptor().getUserPayload(), + groupInputSpec.getGroupName(), groupInputsMap, inputReadyTracker, localDirs, this); + List inputs = Lists.newArrayListWithCapacity(groupInputSpec.getGroupVertices().size()); + for (String groupVertex : groupInputSpec.getGroupVertices()) { + inputs.add(inputsMap.get(groupVertex)); + } + + MergedLogicalInput groupInput = + (MergedLogicalInput) createMergedInput(groupInputSpec.getMergedInputDescriptor(), + mergedInputContext, inputs); groupInputsMap.put(groupInputSpec.getGroupName(), groupInput); } @@ -597,7 +592,7 @@ private void initializeLogicalIOProcessor() throws Exception { } private InputContext createInputContext(Map inputMap, - InputSpec inputSpec, int inputIndex) { + InputSpec inputSpec, int inputIndex) { InputContext inputContext = new TezInputContextImpl(tezConf, localDirs, appAttemptNumber, tezUmbilical, taskSpec.getDAGName(), taskSpec.getVertexName(), @@ -688,8 +683,8 @@ private AbstractLogicalIOProcessor createProcessor( } private void sendTaskGeneratedEvents(List events, - EventProducerConsumerType generator, String taskVertexName, - String edgeVertexName, TezTaskAttemptID taskAttemptID) { + EventProducerConsumerType generator, String taskVertexName, + String edgeVertexName, TezTaskAttemptID taskAttemptID) { if (events == null || events.isEmpty()) { return; } @@ -720,32 +715,32 @@ private boolean handleEvent(TezEvent e) { } try { switch (e.getDestinationInfo().getEventGenerator()) { - case INPUT: - LogicalInput input = inputsMap.get( - e.getDestinationInfo().getEdgeVertexName()); - if (input != null) { - ((InputFrameworkInterface)input).handleEvents(Collections.singletonList(e.getEvent())); - } else { - throw new TezUncheckedException("Unhandled event for invalid target: " - + e); - } - break; - case OUTPUT: - LogicalOutput output = outputsMap.get( - e.getDestinationInfo().getEdgeVertexName()); - if (output != null) { - ((OutputFrameworkInterface)output).handleEvents(Collections.singletonList(e.getEvent())); - } else { - throw new TezUncheckedException("Unhandled event for invalid target: " - + e); - } - break; - case PROCESSOR: - processor.handleEvents(Collections.singletonList(e.getEvent())); - break; - case SYSTEM: - LOG.warn("Trying to send a System event in a Task: " + e); - break; + case INPUT: + LogicalInput input = inputsMap.get( + e.getDestinationInfo().getEdgeVertexName()); + if (input != null) { + ((InputFrameworkInterface) input).handleEvents(Collections.singletonList(e.getEvent())); + } else { + throw new TezUncheckedException("Unhandled event for invalid target: " + + e); + } + break; + case OUTPUT: + LogicalOutput output = outputsMap.get( + e.getDestinationInfo().getEdgeVertexName()); + if (output != null) { + ((OutputFrameworkInterface) output).handleEvents(Collections.singletonList(e.getEvent())); + } else { + throw new TezUncheckedException("Unhandled event for invalid target: " + + e); + } + break; + case PROCESSOR: + processor.handleEvents(Collections.singletonList(e.getEvent())); + break; + case SYSTEM: + LOG.warn("Trying to send a System event in a Task: " + e); + break; } } catch (Throwable t) { LOG.warn("Failed to handle event", t); @@ -839,7 +834,7 @@ private void closeContext(Map contextMap) throws return; } - for(TaskContext context : contextMap.values()) { + for (TaskContext context : contextMap.values()) { closeContext(context); } contextMap.clear(); @@ -892,7 +887,7 @@ public void cleanup() throws InterruptedException { String srcVertexName = entry.getKey(); inputIterator.remove(); try { - ((InputFrameworkInterface)entry.getValue()).close(); + ((InputFrameworkInterface) entry.getValue()).close(); maybeResetInterruptStatus(); } catch (InterruptedException ie) { //reset the status @@ -944,7 +939,7 @@ public void cleanup() throws InterruptedException { processor.close(); LOG.info("Closed processor for vertex={}, index={}, interruptedStatus={}", processor - .getContext().getTaskVertexName(), + .getContext().getTaskVertexName(), processor.getContext().getTaskVertexIndex(), Thread.currentThread().isInterrupted()); maybeResetInterruptStatus(); @@ -955,7 +950,7 @@ public void cleanup() throws InterruptedException { } catch (Throwable e) { LOG.warn( "Ignoring Exception when closing processor(cleanup). Exception class={}, message={}" + - e.getClass().getName(), e.getMessage()); + e.getClass().getName(), e.getMessage()); } } @@ -1007,7 +1002,6 @@ private void cleanupStructures() { objectRegistry = null; } - /** * Print all threads in JVM (only for debugging) */ @@ -1025,13 +1019,13 @@ void printThreads() { } } } - + @Private @VisibleForTesting public Collection getInputContexts() { return this.inputContextMap.values(); } - + @Private @VisibleForTesting public Collection getOutputContexts() { @@ -1043,7 +1037,7 @@ public Collection getOutputContexts() { public ProcessorContext getProcessorContext() { return this.processorContext; } - + @Private @VisibleForTesting public LogicalIOProcessor getProcessor() { diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/RuntimeTask.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/RuntimeTask.java index a53d0d2e7e..2f1111f5f7 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/RuntimeTask.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/RuntimeTask.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -42,7 +42,7 @@ public abstract class RuntimeTask { protected float progress; protected final TezCounters tezCounters; private final Map counterMap = Maps.newConcurrentMap(); - + protected final TaskSpec taskSpec; protected final Configuration tezConf; protected final TezUmbilical tezUmbilical; @@ -55,7 +55,7 @@ public abstract class RuntimeTask { private final AtomicBoolean progressNotified = new AtomicBoolean(false); protected RuntimeTask(TaskSpec taskSpec, Configuration tezConf, - TezUmbilical tezUmbilical, String pid, boolean setupSysCounterUpdater) { + TezUmbilical tezUmbilical, String pid, boolean setupSysCounterUpdater) { this.taskSpec = taskSpec; this.tezConf = tezConf; this.tezUmbilical = tezUmbilical; @@ -88,11 +88,11 @@ public TezCounters addAndGetTezCounter(String name) { counterMap.put(name, counter); return counter; } - + public boolean hasInitialized() { return EnumSet.of(State.RUNNING, State.CLOSED).contains(state.get()); } - + public String getVertexName() { return taskSpec.getVertexName(); } @@ -100,11 +100,11 @@ public String getVertexName() { public void registerError() { errorReported.set(true); } - + public final void notifyProgressInvocation() { progressNotified.lazySet(true); } - + public boolean getAndClearProgressNotification() { boolean retVal = progressNotified.getAndSet(false); return retVal; @@ -146,19 +146,19 @@ public TezTaskAttemptID getTaskAttemptID() { public int getEventCounter() { return eventCounter.get(); } - + public int getNextFromEventId() { return nextFromEventId.get(); } - + public int getNextPreRoutedEventId() { return nextPreRoutedEventId.get(); } - + public void setNextFromEventId(int nextFromEventId) { this.nextFromEventId.set(nextFromEventId); } - + public void setNextPreRoutedEventId(int nextPreRoutedEventId) { this.nextPreRoutedEventId.set(nextPreRoutedEventId); } @@ -166,7 +166,7 @@ public void setNextPreRoutedEventId(int nextPreRoutedEventId) { public boolean isTaskDone() { return taskDone.get(); } - + public void setFrameworkCounters() { if (counterUpdater != null) { this.counterUpdater.updateCounters(); diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/events/TaskAttemptCompletedEvent.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/events/TaskAttemptCompletedEvent.java index 597718f51e..a37c0b15d1 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/events/TaskAttemptCompletedEvent.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/events/TaskAttemptCompletedEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,5 +24,4 @@ public class TaskAttemptCompletedEvent extends Event { public TaskAttemptCompletedEvent() { } - } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/events/TaskAttemptFailedEvent.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/events/TaskAttemptFailedEvent.java index 5606663e8c..bf404dff2b 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/events/TaskAttemptFailedEvent.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/events/TaskAttemptFailedEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -38,5 +38,4 @@ public String getDiagnostics() { public TaskFailureType getTaskFailureType() { return taskFailureType; } - } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/events/TaskAttemptKilledEvent.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/events/TaskAttemptKilledEvent.java index 3f5b32627c..576943c49c 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/events/TaskAttemptKilledEvent.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/events/TaskAttemptKilledEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -30,5 +30,4 @@ public TaskAttemptKilledEvent(String diagnostics) { public String getDiagnostics() { return diagnostics; } - } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/events/TaskStatusUpdateEvent.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/events/TaskStatusUpdateEvent.java index 518cbf4c1b..c505a97dbe 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/events/TaskStatusUpdateEvent.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/events/TaskStatusUpdateEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -37,8 +37,8 @@ public class TaskStatusUpdateEvent extends Event implements Writable { public TaskStatusUpdateEvent() { } - public TaskStatusUpdateEvent(TezCounters tezCounters, float progress, TaskStatistics statistics, - boolean progressNotified) { + public TaskStatusUpdateEvent(TezCounters tezCounters, float progress, TaskStatistics statistics, + boolean progressNotified) { this.tezCounters = tezCounters; this.progress = progress; this.statistics = statistics; @@ -52,11 +52,11 @@ public TezCounters getCounters() { public float getProgress() { return progress; } - + public TaskStatistics getStatistics() { return statistics; } - + public boolean getProgressNotified() { return progressNotified; } @@ -92,5 +92,4 @@ public void readFields(DataInput in) throws IOException { statistics.readFields(in); } } - } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/EventMetaData.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/EventMetaData.java index abe8c9dd53..be7c191269 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/EventMetaData.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/EventMetaData.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,8 +18,6 @@ package org.apache.tez.runtime.api.impl; - - import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -70,8 +68,8 @@ public EventMetaData() { } public EventMetaData(EventProducerConsumerType generator, - String taskVertexName, @Nullable String edgeVertexName, - @Nullable TezTaskAttemptID taskAttemptID) { + String taskVertexName, @Nullable String edgeVertexName, + @Nullable TezTaskAttemptID taskAttemptID) { Objects.requireNonNull(generator, "generator is null"); Objects.requireNonNull(taskVertexName, "taskVertexName is null"); this.producerConsumerType = generator; @@ -112,7 +110,7 @@ public void write(DataOutput out) throws IOException { } else { out.writeBoolean(false); } - if(taskAttemptID != null) { + if (taskAttemptID != null) { out.writeBoolean(true); taskAttemptID.write(out); } else { @@ -139,7 +137,7 @@ public String toString() { return "{ producerConsumerType=" + producerConsumerType + ", taskVertexName=" + taskVertexName + ", edgeVertexName=" + edgeVertexName - + ", taskAttemptId=" + (taskAttemptID == null? "null" : taskAttemptID) + + ", taskAttemptId=" + (taskAttemptID == null ? "null" : taskAttemptID) + " }"; } @@ -187,5 +185,4 @@ public boolean equals(Object obj) { return false; return true; } - } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/EventType.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/EventType.java index 7e365b1870..be1fddda39 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/EventType.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/EventType.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/GroupInputSpec.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/GroupInputSpec.java index 0177bc84fa..62e3a8a5f7 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/GroupInputSpec.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/GroupInputSpec.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -30,34 +30,34 @@ import com.google.common.collect.Lists; -public class GroupInputSpec implements Writable{ +public class GroupInputSpec implements Writable { private String groupName; private List groupVertices; private InputDescriptor mergedInputDescriptor; - + public GroupInputSpec() { // for Writable } - + public String getGroupName() { return groupName; } - + public List getGroupVertices() { return groupVertices; } - + public InputDescriptor getMergedInputDescriptor() { return mergedInputDescriptor; } - + public GroupInputSpec(String groupName, List groupVertices, InputDescriptor inputDescriptor) { this.groupName = StringInterner.intern(groupName); this.groupVertices = groupVertices; this.mergedInputDescriptor = inputDescriptor; } - + @Override public void write(DataOutput out) throws IOException { Text.writeString(out, groupName); @@ -73,22 +73,21 @@ public void readFields(DataInput in) throws IOException { groupName = StringInterner.intern(Text.readString(in)); int numMembers = in.readInt(); groupVertices = Lists.newArrayListWithCapacity(numMembers); - for (int i=0; i * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -26,23 +26,23 @@ public class IOStatistics implements Writable { private volatile long dataSize = 0; private volatile long numItems = 0; - + public void setDataSize(long size) { this.dataSize = size; } - + public long getDataSize() { return dataSize; } - + public void setItemsProcessed(long items) { this.numItems = items; } - + public long getItemsProcessed() { return numItems; } - + public void mergeFrom(org.apache.tez.runtime.api.impl.IOStatistics other) { this.setDataSize(this.getDataSize() + other.getDataSize()); this.setItemsProcessed(this.getItemsProcessed() + other.getItemsProcessed()); @@ -59,5 +59,4 @@ public void readFields(DataInput in) throws IOException { setDataSize(in.readLong()); setItemsProcessed(in.readLong()); } - } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/InputSpec.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/InputSpec.java index f2a23ec5f0..4a0580c6f0 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/InputSpec.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/InputSpec.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,7 +36,7 @@ public InputSpec() { } public InputSpec(String sourceVertexName, InputDescriptor inputDescriptor, - int physicalEdgeCount) { + int physicalEdgeCount) { this.sourceVertexName = StringInterner.intern(sourceVertexName); this.inputDescriptor = inputDescriptor; this.physicalEdgeCount = physicalEdgeCount; @@ -80,5 +80,4 @@ public String toString() { + ", inputClassName=" + inputDescriptor.getClassName() + " }"; } - } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/OutputSpec.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/OutputSpec.java index 9f160052e5..df54c137e6 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/OutputSpec.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/OutputSpec.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,7 +36,7 @@ public OutputSpec() { } public OutputSpec(String destinationVertexName, - OutputDescriptor outputDescriptor, int physicalEdgeCount) { + OutputDescriptor outputDescriptor, int physicalEdgeCount) { this.destinationVertexName = StringInterner.intern(destinationVertexName); this.outputDescriptor = outputDescriptor; this.physicalEdgeCount = physicalEdgeCount; diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TaskSpec.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TaskSpec.java index 15e756b853..d5f246550e 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TaskSpec.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TaskSpec.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -17,7 +17,6 @@ package org.apache.tez.runtime.api.impl; - import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -50,11 +49,11 @@ public class TaskSpec implements Writable, TaskAttemptIDAware { public TaskSpec() { } - + public static TaskSpec createBaseTaskSpec(String dagName, String vertexName, - int vertexParallelism, ProcessorDescriptor processorDescriptor, - List inputSpecList, List outputSpecList, - @Nullable List groupInputSpecList, Configuration taskConf) { + int vertexParallelism, ProcessorDescriptor processorDescriptor, + List inputSpecList, List outputSpecList, + @Nullable List groupInputSpecList, Configuration taskConf) { return new TaskSpec(dagName, vertexName, vertexParallelism, processorDescriptor, inputSpecList, outputSpecList, groupInputSpecList, taskConf); } @@ -102,11 +101,11 @@ public TaskSpec(TezTaskAttemptID taskAttemptID, } public TaskSpec(TezTaskAttemptID taskAttemptID, - String dagName, String vertexName, - int vertexParallelism, - ProcessorDescriptor processorDescriptor, - List inputSpecList, List outputSpecList, - @Nullable List groupInputSpecList, Configuration taskConf) { + String dagName, String vertexName, + int vertexParallelism, + ProcessorDescriptor processorDescriptor, + List inputSpecList, List outputSpecList, + @Nullable List groupInputSpecList, Configuration taskConf) { Objects.requireNonNull(taskAttemptID, "taskAttemptID is null"); Objects.requireNonNull(dagName, "dagName is null"); Objects.requireNonNull(vertexName, "vertexName is null"); @@ -156,7 +155,7 @@ public List getInputs() { public List getOutputs() { return outputSpecList; } - + public List getGroupInputs() { return groupInputSpecList; } @@ -224,7 +223,7 @@ public void readFields(DataInput in) throws IOException { if (hasGroupInputs) { int numGroups = in.readInt(); groupInputSpecList = Lists.newArrayListWithCapacity(numGroups); - for (int i=0; i * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -38,12 +38,12 @@ public class TaskStatistics implements Writable { public void addIO(String edgeName) { addIO(edgeName, new IOStatistics()); } - + public void addIO(String edgeName, IOStatistics stats) { Objects.requireNonNull(stats, edgeName); ioStatistics.put(StringInterner.intern(edgeName), stats); } - + public Map getIOStatistics() { return ioStatistics; } @@ -62,7 +62,7 @@ public void write(DataOutput out) throws IOException { @Override public void readFields(DataInput in) throws IOException { int numEntries = in.readInt(); - for (int i=0; i + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,7 +34,7 @@ public class TezCountersDelegate extends TezCounters { private final TezCounters original; public TezCountersDelegate(TezCounters original, String taskVertexName, String edgeVertexName, - String type) { + String type) { this.original = original; this.groupModifier = TezUtilsInternal.cleanVertexName(taskVertexName) + "_" + type + "_" + TezUtilsInternal.cleanVertexName(edgeVertexName); @@ -47,7 +47,7 @@ public TezCounter findCounter(String groupName, String counterName) { String simpleGroupName; if (groupName.equals(TaskCounter.class.getName())) { simpleGroupName = TaskCounter.class.getSimpleName(); - } else { + } else { simpleGroupName = groupName; } String modifiedGroupName = simpleGroupName + "_" + this.groupModifier; diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezEvent.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezEvent.java index f96a437a49..cd373f9692 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezEvent.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -162,73 +162,73 @@ private void serializeEvent(DataOutput out) throws IOException { } else { AbstractMessage message; switch (eventType) { - case CUSTOM_PROCESSOR_EVENT: - message = - ProtoConverters.convertCustomProcessorEventToProto( - (CustomProcessorEvent) event); - break; - case DATA_MOVEMENT_EVENT: - message = - ProtoConverters.convertDataMovementEventToProto( - (DataMovementEvent) event); - break; - case COMPOSITE_ROUTED_DATA_MOVEMENT_EVENT: + case CUSTOM_PROCESSOR_EVENT: message = - ProtoConverters.convertCompositeRoutedDataMovementEventToProto( - (CompositeRoutedDataMovementEvent) event); - break; - case COMPOSITE_DATA_MOVEMENT_EVENT: - message = - ProtoConverters.convertCompositeDataMovementEventToProto( - (CompositeDataMovementEvent) event); - break; - case VERTEX_MANAGER_EVENT: - message = ProtoConverters.convertVertexManagerEventToProto((VertexManagerEvent) event); - break; - case INPUT_READ_ERROR_EVENT: - InputReadErrorEvent ideEvt = (InputReadErrorEvent) event; - message = InputReadErrorEventProto.newBuilder() - .setIndex(ideEvt.getIndex()) - .setDiagnostics(ideEvt.getDiagnostics()) - .setVersion(ideEvt.getVersion()) - .setIsLocalFetch(ideEvt.isLocalFetch()) - .setIsDiskErrorAtSource(ideEvt.isDiskErrorAtSource()) - .setDestinationLocalhostName(ideEvt.getDestinationLocalhostName()) - .build(); - break; - case TASK_ATTEMPT_FAILED_EVENT: - TaskAttemptFailedEvent tfEvt = (TaskAttemptFailedEvent) event; - message = TaskAttemptFailedEventProto.newBuilder() - .setDiagnostics(tfEvt.getDiagnostics()) - .setTaskFailureType(TezConverterUtils.failureTypeToProto(tfEvt.getTaskFailureType())) - .build(); - break; + ProtoConverters.convertCustomProcessorEventToProto( + (CustomProcessorEvent) event); + break; + case DATA_MOVEMENT_EVENT: + message = + ProtoConverters.convertDataMovementEventToProto( + (DataMovementEvent) event); + break; + case COMPOSITE_ROUTED_DATA_MOVEMENT_EVENT: + message = + ProtoConverters.convertCompositeRoutedDataMovementEventToProto( + (CompositeRoutedDataMovementEvent) event); + break; + case COMPOSITE_DATA_MOVEMENT_EVENT: + message = + ProtoConverters.convertCompositeDataMovementEventToProto( + (CompositeDataMovementEvent) event); + break; + case VERTEX_MANAGER_EVENT: + message = ProtoConverters.convertVertexManagerEventToProto((VertexManagerEvent) event); + break; + case INPUT_READ_ERROR_EVENT: + InputReadErrorEvent ideEvt = (InputReadErrorEvent) event; + message = InputReadErrorEventProto.newBuilder() + .setIndex(ideEvt.getIndex()) + .setDiagnostics(ideEvt.getDiagnostics()) + .setVersion(ideEvt.getVersion()) + .setIsLocalFetch(ideEvt.isLocalFetch()) + .setIsDiskErrorAtSource(ideEvt.isDiskErrorAtSource()) + .setDestinationLocalhostName(ideEvt.getDestinationLocalhostName()) + .build(); + break; + case TASK_ATTEMPT_FAILED_EVENT: + TaskAttemptFailedEvent tfEvt = (TaskAttemptFailedEvent) event; + message = TaskAttemptFailedEventProto.newBuilder() + .setDiagnostics(tfEvt.getDiagnostics()) + .setTaskFailureType(TezConverterUtils.failureTypeToProto(tfEvt.getTaskFailureType())) + .build(); + break; case TASK_ATTEMPT_KILLED_EVENT: TaskAttemptKilledEvent tkEvent = (TaskAttemptKilledEvent) event; message = TaskAttemptKilledEventProto.newBuilder() .setDiagnostics(tkEvent.getDiagnostics()).build(); break; - case TASK_ATTEMPT_COMPLETED_EVENT: - message = TaskAttemptCompletedEventProto.newBuilder() - .build(); - break; - case INPUT_FAILED_EVENT: - InputFailedEvent ifEvt = (InputFailedEvent) event; - message = InputFailedEventProto.newBuilder() - .setTargetIndex(ifEvt.getTargetIndex()) - .setVersion(ifEvt.getVersion()).build(); - break; - case ROOT_INPUT_DATA_INFORMATION_EVENT: - message = ProtoConverters.convertRootInputDataInformationEventToProto( - (InputDataInformationEvent) event); - break; - case ROOT_INPUT_INITIALIZER_EVENT: - message = ProtoConverters - .convertRootInputInitializerEventToProto((InputInitializerEvent) event); - break; - default: - throw new TezUncheckedException("Unknown TezEvent" - + ", type=" + eventType); + case TASK_ATTEMPT_COMPLETED_EVENT: + message = TaskAttemptCompletedEventProto.newBuilder() + .build(); + break; + case INPUT_FAILED_EVENT: + InputFailedEvent ifEvt = (InputFailedEvent) event; + message = InputFailedEventProto.newBuilder() + .setTargetIndex(ifEvt.getTargetIndex()) + .setVersion(ifEvt.getVersion()).build(); + break; + case ROOT_INPUT_DATA_INFORMATION_EVENT: + message = ProtoConverters.convertRootInputDataInformationEventToProto( + (InputDataInformationEvent) event); + break; + case ROOT_INPUT_INITIALIZER_EVENT: + message = ProtoConverters + .convertRootInputInitializerEventToProto((InputInitializerEvent) event); + break; + default: + throw new TezUncheckedException("Unknown TezEvent" + + ", type=" + eventType); } if (out instanceof OutputStream) { //DataOutputBuffer extends DataOutputStream int serializedSize = message.getSerializedSize(); @@ -244,7 +244,6 @@ private void serializeEvent(DataOutput out) throws IOException { out.writeInt(eventBytes.length); out.write(eventBytes); } - } } @@ -258,14 +257,14 @@ private void deserializeEvent(DataInput in) throws IOException { if (eventType.equals(EventType.TASK_STATUS_UPDATE_EVENT)) { // TODO NEWTEZ convert to PB event = new TaskStatusUpdateEvent(); - ((TaskStatusUpdateEvent)event).readFields(in); + ((TaskStatusUpdateEvent) event).readFields(in); } else { int eventBytesLen = in.readInt(); byte[] eventBytes; CodedInputStream input; int startOffset = 0; if (in instanceof DataInputBuffer) { - eventBytes = ((DataInputBuffer)in).getData(); + eventBytes = ((DataInputBuffer) in).getData(); startOffset = ((DataInputBuffer) in).getPosition(); } else { eventBytes = new byte[eventBytesLen]; @@ -273,66 +272,66 @@ private void deserializeEvent(DataInput in) throws IOException { } input = CodedInputStream.newInstance(eventBytes, startOffset, eventBytesLen); switch (eventType) { - case CUSTOM_PROCESSOR_EVENT: - CustomProcessorEventProto cpProto = - CustomProcessorEventProto.parseFrom(input); - event = ProtoConverters.convertCustomProcessorEventFromProto(cpProto); - break; - case DATA_MOVEMENT_EVENT: - DataMovementEventProto dmProto = - DataMovementEventProto.parseFrom(input); - event = ProtoConverters.convertDataMovementEventFromProto(dmProto); - break; - case COMPOSITE_ROUTED_DATA_MOVEMENT_EVENT: - CompositeRoutedDataMovementEventProto edmProto = - CompositeRoutedDataMovementEventProto.parseFrom(eventBytes); - event = ProtoConverters.convertCompositeRoutedDataMovementEventFromProto(edmProto); - break; - case COMPOSITE_DATA_MOVEMENT_EVENT: - CompositeEventProto cProto = CompositeEventProto.parseFrom(input); - event = ProtoConverters.convertCompositeDataMovementEventFromProto(cProto); - break; - case VERTEX_MANAGER_EVENT: - VertexManagerEventProto vmProto = VertexManagerEventProto.parseFrom(input); - event = ProtoConverters.convertVertexManagerEventFromProto(vmProto); - break; - case INPUT_READ_ERROR_EVENT: - InputReadErrorEventProto ideProto = InputReadErrorEventProto.parseFrom(input); - event = InputReadErrorEvent.create(ideProto.getDiagnostics(), ideProto.getIndex(), - ideProto.getVersion(), ideProto.getIsLocalFetch(), ideProto.getIsDiskErrorAtSource(), - ideProto.getDestinationLocalhostName()); - break; - case TASK_ATTEMPT_FAILED_EVENT: - TaskAttemptFailedEventProto tfProto = - TaskAttemptFailedEventProto.parseFrom(input); - event = new TaskAttemptFailedEvent(tfProto.getDiagnostics(), - TezConverterUtils.failureTypeFromProto(tfProto.getTaskFailureType())); - break; - case TASK_ATTEMPT_KILLED_EVENT: - TaskAttemptKilledEventProto tkProto = TaskAttemptKilledEventProto.parseFrom(input); - event = new TaskAttemptKilledEvent(tkProto.getDiagnostics()); - break; - case TASK_ATTEMPT_COMPLETED_EVENT: - event = new TaskAttemptCompletedEvent(); - break; - case INPUT_FAILED_EVENT: - InputFailedEventProto ifProto = - InputFailedEventProto.parseFrom(input); - event = InputFailedEvent.create(ifProto.getTargetIndex(), ifProto.getVersion()); - break; - case ROOT_INPUT_DATA_INFORMATION_EVENT: - RootInputDataInformationEventProto difProto = RootInputDataInformationEventProto - .parseFrom(input); - event = ProtoConverters.convertRootInputDataInformationEventFromProto(difProto); - break; - case ROOT_INPUT_INITIALIZER_EVENT: - EventProtos.RootInputInitializerEventProto riiProto = EventProtos.RootInputInitializerEventProto.parseFrom(input); - event = ProtoConverters.convertRootInputInitializerEventFromProto(riiProto); - break; - default: - // RootInputUpdatePayload event not wrapped in a TezEvent. - throw new TezUncheckedException("Unexpected TezEvent" - + ", type=" + eventType); + case CUSTOM_PROCESSOR_EVENT: + CustomProcessorEventProto cpProto = + CustomProcessorEventProto.parseFrom(input); + event = ProtoConverters.convertCustomProcessorEventFromProto(cpProto); + break; + case DATA_MOVEMENT_EVENT: + DataMovementEventProto dmProto = + DataMovementEventProto.parseFrom(input); + event = ProtoConverters.convertDataMovementEventFromProto(dmProto); + break; + case COMPOSITE_ROUTED_DATA_MOVEMENT_EVENT: + CompositeRoutedDataMovementEventProto edmProto = + CompositeRoutedDataMovementEventProto.parseFrom(eventBytes); + event = ProtoConverters.convertCompositeRoutedDataMovementEventFromProto(edmProto); + break; + case COMPOSITE_DATA_MOVEMENT_EVENT: + CompositeEventProto cProto = CompositeEventProto.parseFrom(input); + event = ProtoConverters.convertCompositeDataMovementEventFromProto(cProto); + break; + case VERTEX_MANAGER_EVENT: + VertexManagerEventProto vmProto = VertexManagerEventProto.parseFrom(input); + event = ProtoConverters.convertVertexManagerEventFromProto(vmProto); + break; + case INPUT_READ_ERROR_EVENT: + InputReadErrorEventProto ideProto = InputReadErrorEventProto.parseFrom(input); + event = InputReadErrorEvent.create(ideProto.getDiagnostics(), ideProto.getIndex(), + ideProto.getVersion(), ideProto.getIsLocalFetch(), ideProto.getIsDiskErrorAtSource(), + ideProto.getDestinationLocalhostName()); + break; + case TASK_ATTEMPT_FAILED_EVENT: + TaskAttemptFailedEventProto tfProto = + TaskAttemptFailedEventProto.parseFrom(input); + event = new TaskAttemptFailedEvent(tfProto.getDiagnostics(), + TezConverterUtils.failureTypeFromProto(tfProto.getTaskFailureType())); + break; + case TASK_ATTEMPT_KILLED_EVENT: + TaskAttemptKilledEventProto tkProto = TaskAttemptKilledEventProto.parseFrom(input); + event = new TaskAttemptKilledEvent(tkProto.getDiagnostics()); + break; + case TASK_ATTEMPT_COMPLETED_EVENT: + event = new TaskAttemptCompletedEvent(); + break; + case INPUT_FAILED_EVENT: + InputFailedEventProto ifProto = + InputFailedEventProto.parseFrom(input); + event = InputFailedEvent.create(ifProto.getTargetIndex(), ifProto.getVersion()); + break; + case ROOT_INPUT_DATA_INFORMATION_EVENT: + RootInputDataInformationEventProto difProto = RootInputDataInformationEventProto + .parseFrom(input); + event = ProtoConverters.convertRootInputDataInformationEventFromProto(difProto); + break; + case ROOT_INPUT_INITIALIZER_EVENT: + EventProtos.RootInputInitializerEventProto riiProto = EventProtos.RootInputInitializerEventProto.parseFrom(input); + event = ProtoConverters.convertRootInputInitializerEventFromProto(riiProto); + break; + default: + // RootInputUpdatePayload event not wrapped in a TezEvent. + throw new TezUncheckedException("Unexpected TezEvent" + + ", type=" + eventType); } if (in instanceof DataInputBuffer) { // Skip so that position is updated diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezHeartbeatRequest.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezHeartbeatRequest.java index fd5bc17521..93aefb82d4 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezHeartbeatRequest.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezHeartbeatRequest.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,7 +29,6 @@ import org.apache.hadoop.io.Writable; import org.apache.tez.dag.records.TezTaskAttemptID; - public class TezHeartbeatRequest implements Writable { private String containerIdentifier; @@ -45,8 +44,8 @@ public TezHeartbeatRequest() { } public TezHeartbeatRequest(long requestId, List events, - int preRoutedStartIndex, String containerIdentifier, - TezTaskAttemptID taskAttemptID, int startIndex, int maxEvents, long usedMemory) { + int preRoutedStartIndex, String containerIdentifier, + TezTaskAttemptID taskAttemptID, int startIndex, int maxEvents, long usedMemory) { this.containerIdentifier = containerIdentifier; this.requestId = requestId; this.events = Collections.unmodifiableList(events); diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezHeartbeatResponse.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezHeartbeatResponse.java index 0aa4db4ec5..d673e6af42 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezHeartbeatResponse.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezHeartbeatResponse.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -53,7 +53,7 @@ public boolean shouldDie() { public long getLastRequestId() { return lastRequestId; } - + public int getNextFromEventId() { return nextFromEventId; } @@ -66,14 +66,14 @@ public void setEvents(List events) { this.events = Collections.unmodifiableList(events); } - public void setLastRequestId(long lastRequestId ) { + public void setLastRequestId(long lastRequestId) { this.lastRequestId = lastRequestId; } public void setShouldDie() { this.shouldDie = true; } - + public void setNextFromEventId(int nextFromEventId) { this.nextFromEventId = nextFromEventId; } @@ -88,7 +88,7 @@ public void write(DataOutput out) throws IOException { out.writeBoolean(shouldDie); out.writeInt(nextFromEventId); out.writeInt(nextPreRoutedEventId); - if(events != null) { + if (events != null) { out.writeBoolean(true); out.writeInt(events.size()); for (TezEvent e : events) { @@ -105,7 +105,7 @@ public void readFields(DataInput in) throws IOException { shouldDie = in.readBoolean(); nextFromEventId = in.readInt(); nextPreRoutedEventId = in.readInt(); - if(in.readBoolean()) { + if (in.readBoolean()) { int eventCount = in.readInt(); events = new ArrayList(eventCount); for (int i = 0; i < eventCount; ++i) { diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java index 1c1c10bf86..2af5f22916 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -61,7 +61,7 @@ public class TezInputContextImpl extends TezTaskContextImpl private final Map inputs; private volatile InputReadyTracker inputReadyTracker; private final InputStatisticsReporterImpl statsReporter; - + class InputStatisticsReporterImpl implements InputStatisticsReporter { @Override @@ -75,15 +75,14 @@ public synchronized void reportDataSize(long size) { public void reportItemsProcessed(long items) { // this is a concurrent map. Plus we are not adding/deleting entries runtimeTask.getTaskStatistics().getIOStatistics().get(sourceVertexName) - .setItemsProcessed(items); + .setItemsProcessed(items); } - } @Private public TezInputContextImpl(Configuration conf, String[] workDirs, int appAttemptNumber, - TezUmbilical tezUmbilical, String dagName, + TezUmbilical tezUmbilical, String dagName, String taskVertexName, String sourceVertexName, int vertexParallelism, TezTaskAttemptID taskAttemptID, int inputIndex, @Nullable UserPayload userPayload, @@ -96,7 +95,7 @@ public TezInputContextImpl(Configuration conf, String[] workDirs, TezExecutors sharedExecutor) { super(conf, workDirs, appAttemptNumber, dagName, taskVertexName, vertexParallelism, taskAttemptID, wrapCounters(runtimeTask, - taskVertexName, sourceVertexName, conf), runtimeTask, tezUmbilical, + taskVertexName, sourceVertexName, conf), runtimeTask, tezUmbilical, serviceConsumerMetadata, auxServiceEnv, memDist, inputDescriptor, objectRegistry, ExecutionContext, memAvailable, sharedExecutor); Objects.requireNonNull(inputIndex, "inputIndex is null"); @@ -116,7 +115,7 @@ vertexParallelism, taskAttemptID, wrapCounters(runtimeTask, } private static TezCounters wrapCounters(LogicalIOProcessorRuntimeTask task, String taskVertexName, - String edgeVertexName, Configuration conf) { + String edgeVertexName, Configuration conf) { TezCounters tezCounters = task.addAndGetTezCounter(edgeVertexName); if (conf.getBoolean(TezConfiguration.TEZ_TASK_GENERATE_COUNTERS_PER_IO, TezConfiguration.TEZ_TASK_GENERATE_COUNTERS_PER_IO_DEFAULT)) { @@ -141,7 +140,7 @@ public void sendEvents(List events) { public UserPayload getUserPayload() { return userPayload; } - + @Override public int getInputIndex() { return inputIndex; diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezMergedInputContextImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezMergedInputContextImpl.java index 553efcd467..1522a2f87b 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezMergedInputContextImpl.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezMergedInputContextImpl.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,8 +18,6 @@ package org.apache.tez.runtime.api.impl; - - import javax.annotation.Nullable; import java.util.Arrays; @@ -32,7 +30,6 @@ import org.apache.tez.runtime.api.MergedLogicalInput; import org.apache.tez.runtime.api.MergedInputContext; - public class TezMergedInputContextImpl implements MergedInputContext { private final UserPayload userPayload; @@ -61,7 +58,7 @@ public TezMergedInputContextImpl(@Nullable UserPayload userPayload, String group public UserPayload getUserPayload() { return userPayload; } - + @Override public void inputIsReady() { inputReadyTracker.setInputIsReady(groupInputsMap.get(groupInputName)); @@ -76,5 +73,4 @@ public String[] getWorkDirs() { public final void notifyProgress() { runtimeTask.notifyProgressInvocation(); } - } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java index a17bc8900d..87dd7dfc36 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -77,31 +77,30 @@ class OutputStatisticsReporterImpl implements OutputStatisticsReporter { public synchronized void reportDataSize(long size) { // this is a concurrent map. Plus we are not adding/deleting entries runtimeTask.getTaskStatistics().getIOStatistics().get(destinationVertexName) - .setDataSize(size); + .setDataSize(size); } @Override public void reportItemsProcessed(long items) { // this is a concurrent map. Plus we are not adding/deleting entries runtimeTask.getTaskStatistics().getIOStatistics().get(destinationVertexName) - .setItemsProcessed(items); + .setItemsProcessed(items); } - } @Private public TezOutputContextImpl(Configuration conf, String[] workDirs, int appAttemptNumber, - TezUmbilical tezUmbilical, String dagName, - String taskVertexName, - String destinationVertexName, - int vertexParallelism, - TezTaskAttemptID taskAttemptID, int outputIndex, - @Nullable UserPayload userPayload, LogicalIOProcessorRuntimeTask runtimeTask, - Map serviceConsumerMetadata, - Map auxServiceEnv, MemoryDistributor memDist, - OutputDescriptor outputDescriptor, ObjectRegistry objectRegistry, - ExecutionContext executionContext, long memAvailable, TezExecutors sharedExecutor) { - super(conf, workDirs, appAttemptNumber, dagName, taskVertexName, + TezUmbilical tezUmbilical, String dagName, + String taskVertexName, + String destinationVertexName, + int vertexParallelism, + TezTaskAttemptID taskAttemptID, int outputIndex, + @Nullable UserPayload userPayload, LogicalIOProcessorRuntimeTask runtimeTask, + Map serviceConsumerMetadata, + Map auxServiceEnv, MemoryDistributor memDist, + OutputDescriptor outputDescriptor, ObjectRegistry objectRegistry, + ExecutionContext executionContext, long memAvailable, TezExecutors sharedExecutor) { + super(conf, workDirs, appAttemptNumber, dagName, taskVertexName, vertexParallelism, taskAttemptID, wrapCounters(runtimeTask, taskVertexName, destinationVertexName, conf), runtimeTask, tezUmbilical, serviceConsumerMetadata, @@ -119,7 +118,7 @@ public TezOutputContextImpl(Configuration conf, String[] workDirs, int appAttemp } private static TezCounters wrapCounters(LogicalIOProcessorRuntimeTask runtimeTask, String taskVertexName, - String edgeVertexName, Configuration conf) { + String edgeVertexName, Configuration conf) { TezCounters tezCounters = runtimeTask.addAndGetTezCounter(edgeVertexName); if (conf.getBoolean(TezConfiguration.TEZ_TASK_GENERATE_COUNTERS_PER_IO, TezConfiguration.TEZ_TASK_GENERATE_COUNTERS_PER_IO_DEFAULT)) { @@ -154,7 +153,6 @@ public String getDestinationVertexName() { return destinationVertexName; } - @Override public String getInputOutputVertexNames() { return String.format("%s -> %s", getTaskVertexName(), getDestinationVertexName()); diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezProcessorContextImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezProcessorContextImpl.java index e09aa8377e..a71b74c40d 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezProcessorContextImpl.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezProcessorContextImpl.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -55,13 +55,13 @@ public class TezProcessorContextImpl extends TezTaskContextImpl implements Proce private final EventMetaData sourceInfo; public TezProcessorContextImpl(Configuration conf, String[] workDirs, int appAttemptNumber, - TezUmbilical tezUmbilical, String dagName, String vertexName, - int vertexParallelism, TezTaskAttemptID taskAttemptID, - @Nullable UserPayload userPayload, LogicalIOProcessorRuntimeTask runtimeTask, - Map serviceConsumerMetadata, - Map auxServiceEnv, MemoryDistributor memDist, - ProcessorDescriptor processorDescriptor, InputReadyTracker inputReadyTracker, ObjectRegistry objectRegistry, - ExecutionContext ExecutionContext, long memAvailable, TezExecutors sharedExecutor) { + TezUmbilical tezUmbilical, String dagName, String vertexName, + int vertexParallelism, TezTaskAttemptID taskAttemptID, + @Nullable UserPayload userPayload, LogicalIOProcessorRuntimeTask runtimeTask, + Map serviceConsumerMetadata, + Map auxServiceEnv, MemoryDistributor memDist, + ProcessorDescriptor processorDescriptor, InputReadyTracker inputReadyTracker, ObjectRegistry objectRegistry, + ExecutionContext ExecutionContext, long memAvailable, TezExecutors sharedExecutor) { super(conf, workDirs, appAttemptNumber, dagName, vertexName, vertexParallelism, taskAttemptID, runtimeTask.addAndGetTezCounter(vertexName), runtimeTask, tezUmbilical, serviceConsumerMetadata, auxServiceEnv, memDist, processorDescriptor, objectRegistry, ExecutionContext, memAvailable, @@ -145,5 +145,4 @@ public void close() throws IOException { this.inputReadyTracker = null; LOG.debug("Cleared TezProcessorContextImpl related information"); } - } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezTaskContextImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezTaskContextImpl.java index b645346654..4f5487dbf8 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezTaskContextImpl.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezTaskContextImpl.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,8 +18,6 @@ package org.apache.tez.runtime.api.impl; - - import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; @@ -75,12 +73,12 @@ public abstract class TezTaskContextImpl implements TaskContext, Closeable { @Private public TezTaskContextImpl(Configuration conf, String[] workDirs, int appAttemptNumber, - String dagName, String taskVertexName, int vertexParallelism, - TezTaskAttemptID taskAttemptID, TezCounters counters, LogicalIOProcessorRuntimeTask runtimeTask, - TezUmbilical tezUmbilical, Map serviceConsumerMetadata, - Map auxServiceEnv, MemoryDistributor memDist, - EntityDescriptor descriptor, ObjectRegistry objectRegistry, - ExecutionContext ExecutionContext, long memAvailable, TezExecutors sharedExecutor) { + String dagName, String taskVertexName, int vertexParallelism, + TezTaskAttemptID taskAttemptID, TezCounters counters, LogicalIOProcessorRuntimeTask runtimeTask, + TezUmbilical tezUmbilical, Map serviceConsumerMetadata, + Map auxServiceEnv, MemoryDistributor memDist, + EntityDescriptor descriptor, ObjectRegistry objectRegistry, + ExecutionContext ExecutionContext, long memAvailable, TezExecutors sharedExecutor) { Objects.requireNonNull(conf, "conf is null"); Objects.requireNonNull(dagName, "dagName is null"); Objects.requireNonNull(taskVertexName, "taskVertexName is null"); @@ -180,7 +178,7 @@ public String[] getWorkDirs() { public String getUniqueIdentifier() { return uniqueIdentifier; } - + @Override public ObjectRegistry getObjectRegistry() { return objectRegistry; @@ -190,7 +188,7 @@ public ObjectRegistry getObjectRegistry() { public final void notifyProgress() { runtimeTask.notifyProgressInvocation(); } - + @Override public ByteBuffer getServiceConsumerMetaData(String serviceName) { return (ByteBuffer) serviceConsumerMetadata.get(serviceName) @@ -214,7 +212,7 @@ public void requestInitialMemory(long size, MemoryUpdateCallback callbackHandler callbackHandler = new MemoryUpdateCallback() { @Override public void memoryAssigned(long assignedSize) { - + } }; } @@ -225,7 +223,7 @@ public void memoryAssigned(long assignedSize) { public long getTotalMemoryAvailableToTask() { return memAvailable; } - + protected void signalFatalError(Throwable t, String message, EventMetaData sourceInfo) { signalFailure(TaskFailureType.NON_FATAL, t, message, sourceInfo); } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezUmbilical.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezUmbilical.java index b606dea554..ff5e7d9b9e 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezUmbilical.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezUmbilical.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,5 +36,4 @@ void signalFailure(TezTaskAttemptID taskAttemptID, TaskFailureType taskFailureTy void signalKillSelf(TezTaskAttemptID taskAttemptID, Throwable t, String message, EventMetaData sourceInfo); boolean canCommit(TezTaskAttemptID taskAttemptID) throws IOException; - } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/objectregistry/ObjectRegistryImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/objectregistry/ObjectRegistryImpl.java index eccb71c1dd..09822443f5 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/objectregistry/ObjectRegistryImpl.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/objectregistry/ObjectRegistryImpl.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -27,7 +27,7 @@ import org.apache.tez.runtime.api.ObjectRegistry; public class ObjectRegistryImpl implements ObjectRegistry { - + public enum ObjectLifeCycle { /** Objects are valid for the lifetime of the Tez JVM/Session */ @@ -44,7 +44,7 @@ public enum ObjectLifeCycle { new HashMap>(); private synchronized Object add(ObjectLifeCycle lifeCycle, - String key, Object value) { + String key, Object value) { Map.Entry oldEntry = objectCache.put(key, new AbstractMap.SimpleImmutableEntry( @@ -66,7 +66,7 @@ public synchronized boolean delete(String key) { public synchronized void clearCache(ObjectLifeCycle lifeCycle) { Iterator>> it = - objectCache.entrySet().iterator(); + objectCache.entrySet().iterator(); while (it.hasNext()) { Entry> entry = it.next(); if (entry.getValue().getValue().equals(lifeCycle)) { @@ -89,5 +89,4 @@ public synchronized Object cacheForDAG(String key, Object value) { public synchronized Object cacheForSession(String key, Object value) { return add(ObjectLifeCycle.SESSION, key, value); } - } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java index d8ba1f71ff..8472a7262e 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -56,7 +56,7 @@ public class MemoryDistributor { private final int numTotalInputs; private final int numTotalOutputs; private final Configuration conf; - + private AtomicInteger numInputsSeen = new AtomicInteger(0); private AtomicInteger numOutputsSeen = new AtomicInteger(0); @@ -101,13 +101,11 @@ public MemoryDistributor(int numTotalInputs, int numTotalOutputs, Configuration + ", allocatorClassName=" + allocatorClassName); } - - /** * Used by the Tez framework to request memory on behalf of user requests. */ public void requestMemory(long requestSize, MemoryUpdateCallback callback, - TaskContext taskContext, EntityDescriptor descriptor) { + TaskContext taskContext, EntityDescriptor descriptor) { registerRequest(requestSize, callback, taskContext, descriptor); } @@ -161,8 +159,6 @@ public Long apply(RequestorInfo requestInfo) { } } - - /** * Allow tests to set memory. * @param size @@ -174,7 +170,7 @@ void setJvmMemory(long size) { } private long registerRequest(long requestSize, MemoryUpdateCallback callback, - TaskContext entityContext, EntityDescriptor descriptor) { + TaskContext entityContext, EntityDescriptor descriptor) { Preconditions.checkArgument(requestSize >= 0); Objects.requireNonNull(callback); Objects.requireNonNull(entityContext); @@ -184,24 +180,24 @@ private long registerRequest(long requestSize, MemoryUpdateCallback callback, "A single entity can only make one call to request resources for now"); } - RequestorInfo requestInfo = new RequestorInfo(entityContext,requestSize, callback, descriptor); + RequestorInfo requestInfo = new RequestorInfo(entityContext, requestSize, callback, descriptor); switch (requestInfo.getRequestContext().getComponentType()) { - case INPUT: - numInputsSeen.incrementAndGet(); - Preconditions.checkState(numInputsSeen.get() <= numTotalInputs, - "Num Requesting Inputs higher than total # of inputs: " + numInputsSeen + ", " - + numTotalInputs); - break; - case OUTPUT: - numOutputsSeen.incrementAndGet(); - Preconditions.checkState(numOutputsSeen.get() <= numTotalOutputs, - "Num Requesting Inputs higher than total # of outputs: " + numOutputsSeen + ", " - + numTotalOutputs); - break; - case PROCESSOR: - break; - default: - break; + case INPUT: + numInputsSeen.incrementAndGet(); + Preconditions.checkState(numInputsSeen.get() <= numTotalInputs, + "Num Requesting Inputs higher than total # of inputs: " + numInputsSeen + ", " + + numTotalInputs); + break; + case OUTPUT: + numOutputsSeen.incrementAndGet(); + Preconditions.checkState(numOutputsSeen.get() <= numTotalOutputs, + "Num Requesting Inputs higher than total # of outputs: " + numOutputsSeen + ", " + + numTotalOutputs); + break; + case PROCESSOR: + break; + default: + break; } requestList.add(requestInfo); return -1; @@ -225,7 +221,6 @@ private void validateAllocations(Iterable allocations, int numRequestors) } } - private static class RequestorInfo { private static final Logger LOG = LoggerFactory.getLogger(RequestorInfo.class); @@ -234,7 +229,7 @@ private static class RequestorInfo { private final InitialMemoryRequestContext requestContext; public RequestorInfo(TaskContext taskContext, long requestSize, - final MemoryUpdateCallback callback, EntityDescriptor descriptor) { + final MemoryUpdateCallback callback, EntityDescriptor descriptor) { InitialMemoryRequestContext.ComponentType type; String componentVertexName; if (taskContext instanceof InputContext) { @@ -264,7 +259,6 @@ public InitialMemoryRequestContext getRequestContext() { } } - private void logInitialRequests(List initialRequests) { if (initialRequests != null && !initialRequests.isEmpty()) { StringBuilder sb = new StringBuilder(); @@ -288,7 +282,7 @@ private void logFinalAllocations(Iterable allocations, List Iterator allocatedIter = allocations.iterator(); StringBuilder sb = new StringBuilder(); - for (int i = 0 ; i < requestList.size() ; i++) { + for (int i = 0; i < requestList.size(); i++) { long allocated = allocatedIter.next(); InitialMemoryRequestContext context = requestList.get(i).getRequestContext(); sb.append("["); @@ -304,5 +298,4 @@ private void logFinalAllocations(Iterable allocations, List LOG.info("Allocations=" + sb.toString()); } } - } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/ScalingAllocator.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/ScalingAllocator.java index e045abd9d2..07ffc55fb7 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/ScalingAllocator.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/ScalingAllocator.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -43,7 +43,7 @@ public class ScalingAllocator implements InitialMemoryAllocator { @Override public Iterable assignMemory(long availableForAllocation, int numTotalInputs, - int numTotalOutputs, Iterable requests) { + int numTotalOutputs, Iterable requests) { int numRequests = 0; long totalRequested = 0; @@ -69,10 +69,10 @@ public Iterable assignMemory(long availableForAllocation, int numTotalInpu // users in this case, keeping Processor, caching etc in mind. return Lists.newArrayList(Iterables.transform(requests, new Function() { - public Long apply(InitialMemoryRequestContext requestContext) { - return requestContext.getRequestedSize(); - } - })); + public Long apply(InitialMemoryRequestContext requestContext) { + return requestContext.getRequestedSize(); + } + })); } List allocations = Lists.newArrayListWithCapacity(numRequests); @@ -84,7 +84,7 @@ public Long apply(InitialMemoryRequestContext requestContext) { } else { long allocated = (long) ((requestedSize / (double) totalRequested) * availableForAllocation); allocations.add(allocated); - LOG.debug("Scaling requested: {} to allocated: {}", requestedSize, allocated); + LOG.debug("Scaling requested: {} to allocated: {}", requestedSize, allocated); } } return allocations; diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/security/JobTokenSelector.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/security/JobTokenSelector.java index 7d15a57944..3afbc8f631 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/security/JobTokenSelector.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/security/JobTokenSelector.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -39,7 +39,7 @@ public class JobTokenSelector implements TokenSelector { @SuppressWarnings("unchecked") @Override public Token selectToken(Text service, - Collection> tokens) { + Collection> tokens) { if (service == null) { return null; } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/internals/api/TaskReporterInterface.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/internals/api/TaskReporterInterface.java index b7d5fb5885..4ccfa2a970 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/internals/api/TaskReporterInterface.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/internals/api/TaskReporterInterface.java @@ -34,9 +34,9 @@ public interface TaskReporterInterface { boolean taskSucceeded(TezTaskAttemptID taskAttemptId) throws IOException, TezException; boolean taskFailed(TezTaskAttemptID taskAttemptId, - TaskFailureType taskFailureType, - Throwable cause, - String diagnostics, EventMetaData srcMeta) throws IOException, + TaskFailureType taskFailureType, + Throwable cause, + String diagnostics, EventMetaData srcMeta) throws IOException, TezException; boolean taskKilled(TezTaskAttemptID taskAttemtpId, Throwable cause, String diagnostics, @@ -47,5 +47,4 @@ boolean taskKilled(TezTaskAttemptID taskAttemtpId, Throwable cause, String diagn boolean canCommit(TezTaskAttemptID taskAttemptId) throws IOException; void shutdown(); - -} \ No newline at end of file +} diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/internals/api/TezTrapEvent.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/internals/api/TezTrapEvent.java index 8c63b74007..9da2b72ca9 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/internals/api/TezTrapEvent.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/internals/api/TezTrapEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -49,4 +49,4 @@ public TezTrapEvent(final List events) { public final List getTezEvents() { return tezEvents; } -} \ No newline at end of file +} diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/internals/api/TezTrapEventType.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/internals/api/TezTrapEventType.java index 89cb78ea06..846b1b30f2 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/internals/api/TezTrapEventType.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/internals/api/TezTrapEventType.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/FileSystemStatisticUpdater.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/FileSystemStatisticUpdater.java index bb15ef159f..dbf1b6375d 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/FileSystemStatisticUpdater.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/FileSystemStatisticUpdater.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java index 48676e225b..7fc3f5c357 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/metrics/TaskCounterUpdater.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -35,7 +35,6 @@ import org.apache.tez.common.counters.TezCounters; import org.apache.tez.dag.api.TezConfiguration; - /** * Updates counters with various task specific statistics. Currently, this * should be invoked only once per task. TODO Eventually - change this so that @@ -52,22 +51,21 @@ public class TaskCounterUpdater { * A Map where Key-> URIScheme and value->FileSystemStatisticUpdater */ private Map statisticUpdaters = - new HashMap(); + new HashMap(); protected final GcTimeUpdater gcUpdater; private ResourceCalculatorProcessTree pTree; private long initCpuCumulativeTime = 0; private final String pid; - + public TaskCounterUpdater(TezCounters counters, Configuration conf, String pid) { this.tezCounters = counters; - this.conf = conf; + this.conf = conf; this.gcUpdater = new GcTimeUpdater(tezCounters); this.pid = pid; initResourceCalculatorPlugin(); recordInitialCpuStats(); } - public void updateCounters() { // FileSystemStatistics are reset each time a new task is seen by the // container. @@ -76,7 +74,7 @@ public void updateCounters() { // Container, and strange values for READ_OPS etc. Map> map = new HashMap>(); - for(Statistics stat: FileSystem.getAllStatistics()) { + for (Statistics stat : FileSystem.getAllStatistics()) { String uriScheme = stat.getScheme(); if (map.containsKey(uriScheme)) { List list = map.get(uriScheme); @@ -87,9 +85,9 @@ public void updateCounters() { map.put(uriScheme, list); } } - for (Map.Entry> entry: map.entrySet()) { + for (Map.Entry> entry : map.entrySet()) { FileSystemStatisticUpdater updater = statisticUpdaters.get(entry.getKey()); - if(updater==null) {//new FileSystem has been found in the cache + if (updater == null) {//new FileSystem has been found in the cache updater = new FileSystemStatisticUpdater(tezCounters, entry.getValue(), entry.getKey()); @@ -101,14 +99,14 @@ public void updateCounters() { gcUpdater.incrementGcCounter(); updateResourceCounters(); } - + private void recordInitialCpuStats() { if (pTree != null) { pTree.updateProcessTree(); initCpuCumulativeTime = pTree.getCumulativeCpuTime(); } } - + /** * Update resource information counters */ @@ -130,7 +128,7 @@ void updateResourceCounters() { tezCounters.findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES).setValue(pMem); tezCounters.findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES).setValue(vMem); } - + /** * Updates the {@link TaskCounter#COMMITTED_HEAP_BYTES} counter to reflect the * current total committed heap space usage of this JVM. @@ -138,14 +136,14 @@ void updateResourceCounters() { private void updateHeapUsageCounter() { long currentHeapUsage = Runtime.getRuntime().totalMemory(); tezCounters.findCounter(TaskCounter.COMMITTED_HEAP_BYTES) - .setValue(currentHeapUsage); + .setValue(currentHeapUsage); } - + private void initResourceCalculatorPlugin() { Class clazz = this.conf.getClass( TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS, TezMxBeanResourceCalculator.class, - ResourceCalculatorProcessTree.class); + ResourceCalculatorProcessTree.class); pTree = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(pid, clazz, conf); diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/ContainerReporter.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/ContainerReporter.java index 0ece2271c5..45b682abcb 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/ContainerReporter.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/ContainerReporter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,7 +31,7 @@ /** * Responsible for communication between a running Container and the ApplicationMaster. The main * functionality is to poll for new tasks. - * + * */ public class ContainerReporter extends CallableWithNdc { @@ -45,7 +45,7 @@ public class ContainerReporter extends CallableWithNdc { private long nextGetTaskPrintTime; ContainerReporter(TezTaskUmbilicalProtocol umbilical, ContainerContext containerContext, - int getTaskMaxSleepTime) { + int getTaskMaxSleepTime) { this.umbilical = umbilical; this.containerContext = containerContext; this.getTaskMaxSleepTime = getTaskMaxSleepTime; @@ -70,7 +70,7 @@ protected ContainerTask callInternal() throws Exception { + " TaskInfo: shouldDie: " + containerTask.shouldDie() + (containerTask.shouldDie() == true ? "" : ", currentTaskAttemptId: " - + containerTask.getTaskSpec().getTaskAttemptID())); + + containerTask.getTaskSpec().getTaskAttemptID())); return containerTask; } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/ErrorReporter.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/ErrorReporter.java index 1146ce468f..cf8819b1c9 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/ErrorReporter.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/ErrorReporter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,6 +21,6 @@ public interface ErrorReporter { void reportError(Throwable t); - + void shutdownRequested(); } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskReporter.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskReporter.java index eeb24343be..e640418b6f 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskReporter.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskReporter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -71,7 +71,7 @@ * Responsible for communication between tasks running in a Container and the ApplicationMaster. * Takes care of sending heartbeats (regular and OOB) to the AM - to send generated events, and to * retrieve events specific to this task. - * + * */ public class TaskReporter implements TaskReporterInterface { @@ -90,7 +90,7 @@ public class TaskReporter implements TaskReporterInterface { HeartbeatCallable currentCallable; public TaskReporter(TezTaskUmbilicalProtocol umbilical, long amPollInterval, - long sendCounterInterval, int maxEventsToGet, AtomicLong requestCounter, String containerIdStr) { + long sendCounterInterval, int maxEventsToGet, AtomicLong requestCounter, String containerIdStr) { this.umbilical = umbilical; this.pollInterval = amPollInterval; this.sendCounterInterval = sendCounterInterval; @@ -107,7 +107,7 @@ public TaskReporter(TezTaskUmbilicalProtocol umbilical, long amPollInterval, */ @Override public synchronized void registerTask(RuntimeTask task, - ErrorReporter errorReporter) { + ErrorReporter errorReporter) { currentCallable = new HeartbeatCallable(task, umbilical, pollInterval, sendCounterInterval, maxEventsToGet, requestCounter, containerIdStr); ListenableFuture future = heartbeatExecutor.submit(currentCallable); @@ -171,13 +171,13 @@ static class HeartbeatCallable implements Callable { private AtomicInteger nonOobHeartbeatCounter = new AtomicInteger(0); private int nextHeartbeatNumToLog = 0; /* - * Tracks the last non-OOB heartbeat number at which counters were sent to the AM. + * Tracks the last non-OOB heartbeat number at which counters were sent to the AM. */ private int prevCounterSendHeartbeatNum = 0; public HeartbeatCallable(RuntimeTask task, - TezTaskUmbilicalProtocol umbilical, long amPollInterval, long sendCounterInterval, - int maxEventsToGet, AtomicLong requestCounter, String containerIdStr) { + TezTaskUmbilicalProtocol umbilical, long amPollInterval, long sendCounterInterval, + int maxEventsToGet, AtomicLong requestCounter, String containerIdStr) { this.pollInterval = amPollInterval; this.sendCounterInterval = sendCounterInterval; @@ -361,7 +361,7 @@ private boolean taskSucceeded(TezTaskAttemptID taskAttemptID) throws IOException return askedToDie.get(); } } - + @VisibleForTesting TaskStatusUpdateEvent getStatusUpdateEvent(boolean sendCounters) { TezCounters counters = null; @@ -461,11 +461,11 @@ public synchronized boolean taskSucceeded(TezTaskAttemptID taskAttemptID) throws @Override public synchronized boolean taskFailed(TezTaskAttemptID taskAttemptID, - TaskFailureType taskFailureType, - Throwable t, String diagnostics, - EventMetaData srcMeta) throws IOException, + TaskFailureType taskFailureType, + Throwable t, String diagnostics, + EventMetaData srcMeta) throws IOException, TezException { - if(!isShuttingDown()) { + if (!isShuttingDown()) { return currentCallable.taskTerminated(taskAttemptID, false, taskFailureType, t, diagnostics, srcMeta); } return false; @@ -474,7 +474,7 @@ public synchronized boolean taskFailed(TezTaskAttemptID taskAttemptID, @Override public boolean taskKilled(TezTaskAttemptID taskAttemptID, Throwable t, String diagnostics, EventMetaData srcMeta) throws IOException, TezException { - if(!isShuttingDown()) { + if (!isShuttingDown()) { return currentCallable.taskTerminated(taskAttemptID, true, null, t, diagnostics, srcMeta); } return false; diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskRunner2Callable.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskRunner2Callable.java index 810a806228..b5674bc8a1 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskRunner2Callable.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TaskRunner2Callable.java @@ -31,9 +31,9 @@ /** * This class is responsible for running a {@link LogicalIOProcessorRuntimeTask}. * It does not worry about reporting errors, heartbeats etc. - * + *

    * Returns success / interrupt / failure status via it's return parameter. - * + *

    * It's the responsibility of the invoker to handle whatever exceptions may be generated by this. */ public class TaskRunner2Callable extends CallableWithNdc { @@ -53,7 +53,7 @@ public class TaskRunner2Callable extends CallableWithNdc + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -127,12 +127,12 @@ public class TezChild { private final TezExecutors sharedExecutor; public TezChild(Configuration conf, String host, int port, String containerIdentifier, - String tokenIdentifier, int appAttemptNumber, String workingDir, String[] localDirs, - Map serviceProviderEnvMap, - ObjectRegistryImpl objectRegistry, String pid, - ExecutionContext executionContext, - Credentials credentials, long memAvailable, String user, TezTaskUmbilicalProtocol umbilical, - boolean updateSysCounters, HadoopShim hadoopShim) throws IOException, InterruptedException { + String tokenIdentifier, int appAttemptNumber, String workingDir, String[] localDirs, + Map serviceProviderEnvMap, + ObjectRegistryImpl objectRegistry, String pid, + ExecutionContext executionContext, + Credentials credentials, long memAvailable, String user, TezTaskUmbilicalProtocol umbilical, + boolean updateSysCounters, HadoopShim hadoopShim) throws IOException, InterruptedException { this.defaultConf = conf; this.containerIdString = containerIdentifier; this.appAttemptNumber = appAttemptNumber; @@ -168,11 +168,10 @@ public TezChild(Configuration conf, String host, int port, String containerIdent this.objectRegistry = objectRegistry; - if (LOG.isDebugEnabled()) { LOG.debug("Executing with tokens:"); for (Token token : credentials.getAllTokens()) { - LOG.debug("",token); + LOG.debug("", token); } } @@ -202,7 +201,7 @@ public TezTaskUmbilicalProtocol run() throws Exception { } TezCommonUtils.logCredentials(LOG, credentials, "tezChildInit"); } - + public ContainerExecutionResult run() throws IOException, InterruptedException, TezException { ContainerContext containerContext = new ContainerContext(containerIdString); @@ -299,7 +298,7 @@ public ContainerExecutionResult run() throws IOException, InterruptedException, /** * Setup - * + * * @param containerTask * the new task specification. Must be a valid task * @param childUGI @@ -307,7 +306,7 @@ public ContainerExecutionResult run() throws IOException, InterruptedException, * @return childUGI */ UserGroupInformation handleNewTaskCredentials(ContainerTask containerTask, - UserGroupInformation childUGI) { + UserGroupInformation childUGI) { // Re-use the UGI only if the Credentials have not changed. Preconditions.checkState(!containerTask.shouldDie()); Preconditions.checkState(containerTask.getTaskSpec() != null); @@ -328,13 +327,13 @@ UserGroupInformation handleNewTaskCredentials(ContainerTask containerTask, /** * Handles any additional resources to be localized for the new task - * + * * @param containerTask * @throws IOException * @throws TezException */ private void handleNewTaskLocalResources(ContainerTask containerTask, - UserGroupInformation ugi) throws IOException, TezException { + UserGroupInformation ugi) throws IOException, TezException { final Map additionalResources = containerTask.getAdditionalResources(); LOG.debug("Additional Resources added to container: {}", additionalResources); @@ -365,7 +364,7 @@ public URI apply(TezLocalResource input) { /** * Cleans entries from the object registry, and resets the startedInputsMap if required - * + * * @param containerTask * the new task specification. Must be a valid task */ @@ -395,7 +394,7 @@ public void shutdown() { for (Runnable r : pendingRunnables) { LOG.info("Cancelling pending runnable ({}) during TezChild shutdown for containerId={}", r.hashCode(), containerIdString); - ((FutureTask)r).cancel(false); + ((FutureTask) r).cancel(false); } if (taskReporter != null) { taskReporter.shutdown(); @@ -431,7 +430,7 @@ public int getExitCode() { private final String errorMessage; public ContainerExecutionResult(ExitStatus exitStatus, @Nullable Throwable throwable, - @Nullable String errorMessage) { + @Nullable String errorMessage) { this.exitStatus = exitStatus; this.throwable = throwable; this.errorMessage = errorMessage; @@ -460,10 +459,10 @@ public String toString() { } public static TezChild newTezChild(Configuration conf, String host, int port, String containerIdentifier, - String tokenIdentifier, int attemptNumber, String[] localDirs, String workingDirectory, - Map serviceProviderEnvMap, @Nullable String pid, - ExecutionContext executionContext, Credentials credentials, long memAvailable, String user, - TezTaskUmbilicalProtocol tezUmbilical, boolean updateSysCounters, HadoopShim hadoopShim) + String tokenIdentifier, int attemptNumber, String[] localDirs, String workingDirectory, + Map serviceProviderEnvMap, @Nullable String pid, + ExecutionContext executionContext, Credentials credentials, long memAvailable, String user, + TezTaskUmbilicalProtocol tezUmbilical, boolean updateSysCounters, HadoopShim hadoopShim) throws IOException, InterruptedException, TezException { // Pull in configuration specified for the session. @@ -489,7 +488,6 @@ public static void main(String[] args) throws IOException, InterruptedException, Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); final String pid = System.getenv().get("JVM_PID"); - assert args.length == 5; String host = args[0]; int port = Integer.parseInt(args[1]); @@ -534,5 +532,4 @@ public static void main(String[] args) throws IOException, InterruptedException, private void handleError(Throwable t) { shutdown(); } - } diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner2.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner2.java index bbf037b8bf..8d1dc779c0 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner2.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTaskRunner2.java @@ -96,7 +96,7 @@ public class TezTaskRunner2 { private final Lock oobSignalLock = new ReentrantLock(); private final Condition oobSignalCondition = oobSignalLock.newCondition(); - private volatile long taskKillStartTime = 0; + private volatile long taskKillStartTime = 0; final Configuration taskConf; private final HadoopShim hadoopShim; @@ -110,14 +110,14 @@ public class TezTaskRunner2 { @Deprecated public TezTaskRunner2(Configuration tezConf, UserGroupInformation ugi, String[] localDirs, - TaskSpec taskSpec, int appAttemptNumber, - Map serviceConsumerMetadata, - Map serviceProviderEnvMap, - Multimap startedInputsMap, - TaskReporterInterface taskReporter, ExecutorService executor, - ObjectRegistry objectRegistry, String pid, - ExecutionContext executionContext, long memAvailable, - boolean updateSysCounters, HadoopShim hadoopShim) throws IOException { + TaskSpec taskSpec, int appAttemptNumber, + Map serviceConsumerMetadata, + Map serviceProviderEnvMap, + Multimap startedInputsMap, + TaskReporterInterface taskReporter, ExecutorService executor, + ObjectRegistry objectRegistry, String pid, + ExecutionContext executionContext, long memAvailable, + boolean updateSysCounters, HadoopShim hadoopShim) throws IOException { this(tezConf, ugi, localDirs, taskSpec, appAttemptNumber, serviceConsumerMetadata, serviceProviderEnvMap, startedInputsMap, taskReporter, executor, objectRegistry, pid, executionContext, memAvailable, updateSysCounters, hadoopShim, null); @@ -157,10 +157,10 @@ public TezTaskRunner2(Configuration tezConf, UserGroupInformation ugi, String[] /** * Throws an exception only when there was a communication error reported by * the TaskReporter. - * + *

    * Otherwise, this takes care of all communication with the AM for a a running task - which * includes informing the AM about Failures and Success. - * + *

    * If a kill request is made to the task, it will not communicate this information to * the AM - since a task KILL is an external event, and whoever invoked it should * be able to track it. @@ -261,7 +261,6 @@ public TaskRunner2Result run() { default: LOG.error("Unexpected EndReason. File a bug"); return logAndReturnEndResult(EndReason.TASK_ERROR, firstTaskFailureType, new RuntimeException("Unexpected EndReason"), stopContainerRequested.get()); - } } finally { // Clear the interrupted status of the blocking thread, in case it is set after the @@ -312,6 +311,7 @@ void processCallableResult(TaskRunner2CallableResult executionResult) { /** * Attempt to kill the running task, if it hasn't already completed for some other reason. + * * @return true if the task kill was honored, false otherwise */ public boolean killTask() { @@ -384,10 +384,8 @@ public void signalFailure(TezTaskAttemptID taskAttemptID, TaskFailureType taskFa public void signalKillSelf(TezTaskAttemptID taskAttemptID, Throwable t, String message, EventMetaData sourceInfo) { signalTerminationInternal(taskAttemptID, EndReason.TASK_KILL_REQUEST, null, t, message, sourceInfo, true); - } - @Override public boolean canCommit(TezTaskAttemptID taskAttemptID) throws IOException { // Task checking whether it can commit. @@ -405,7 +403,6 @@ public boolean canCommit(TezTaskAttemptID taskAttemptID) throws IOException { } } - @Override public void reportError(Throwable t) { // Umbilical reporting an error during heartbeat @@ -453,7 +450,6 @@ public void shutdownRequested() { } } - private void signalTerminationInternal(TezTaskAttemptID taskAttemptID, EndReason endReason, TaskFailureType taskFailureType, Throwable t, String message, EventMetaData sourceInfo, boolean isKill) { @@ -527,7 +523,6 @@ private synchronized boolean trySettingEndReason(EndReason endReason) { return false; } - private void registerFirstException(TaskFailureType taskFailureType, Throwable t, EventMetaData sourceInfo) { Preconditions.checkState(isRunningState()); errorSeen.set(true); @@ -536,7 +531,6 @@ private void registerFirstException(TaskFailureType taskFailureType, Throwable t this.exceptionSourceInfo = sourceInfo; } - private String getTaskDiagnosticsString(Throwable t, String message, String typeString) { String diagnostics; if (t != null && message != null) { @@ -579,4 +573,4 @@ private void logAborting(String abortReason) { LOG.info("Attempting to abort {} due to an invocation of {}", task.getTaskAttemptID(), abortReason); } -} \ No newline at end of file +} diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTrapEventHandler.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTrapEventHandler.java index b35dbb0162..6421b09b6c 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTrapEventHandler.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezTrapEventHandler.java @@ -12,7 +12,6 @@ * limitations under the License. */ - package org.apache.tez.runtime.task; import com.google.common.base.Preconditions; @@ -52,17 +51,18 @@ public class TezTrapEventHandler implements EventHandler { private final TezUmbilical tezUmbilical; /** - * @param output context that will report the events. + * @param output context that will report the events. * @param umbilical used to send the events to the AM. */ TezTrapEventHandler(final OutputContext output, - final TezUmbilical umbilical) { + final TezUmbilical umbilical) { this.outputContext = output; this.tezUmbilical = umbilical; } /** * Decide what to do with the events. + * * @param tezTrapEvent event holding the tez events. */ @Override @@ -70,23 +70,23 @@ public final void handle(final TezTrapEvent tezTrapEvent) { Preconditions.checkArgument(tezTrapEvent.getTezEvents() != null); List tezEvents = new ArrayList( tezTrapEvent.getTezEvents().size()); - for (TezEvent tezEvent: tezTrapEvent.getTezEvents()) { + for (TezEvent tezEvent : tezTrapEvent.getTezEvents()) { switch (tezEvent.getEventType()) { - case COMPOSITE_DATA_MOVEMENT_EVENT: - case DATA_MOVEMENT_EVENT: - String errorMsg = "Some events won't be sent to the AM because all" - + " the events should have been sent at this point. Most likely" - + " this would result in a bug. " - + " event:" + tezEvent.toString(); - Throwable throwable = new Throwable(errorMsg); - LOG.error(errorMsg, throwable); - break; - default: - LOG.info("Event of type " + tezEvent.getEventType() + " will be sent" - + " to the AM after the task was closed "); - tezEvents.add(tezEvent); + case COMPOSITE_DATA_MOVEMENT_EVENT: + case DATA_MOVEMENT_EVENT: + String errorMsg = "Some events won't be sent to the AM because all" + + " the events should have been sent at this point. Most likely" + + " this would result in a bug. " + + " event:" + tezEvent.toString(); + Throwable throwable = new Throwable(errorMsg); + LOG.error(errorMsg, throwable); + break; + default: + LOG.info("Event of type " + tezEvent.getEventType() + " will be sent" + + " to the AM after the task was closed "); + tezEvents.add(tezEvent); } } tezUmbilical.addEvents(tezEvents); } -} \ No newline at end of file +} diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestInputReadyTracker.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestInputReadyTracker.java index e10c83f9a5..98e88c06ee 100644 --- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestInputReadyTracker.java +++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestInputReadyTracker.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -41,11 +41,10 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; - public class TestInputReadyTracker { private static final long SLEEP_TIME = 200l; - + @Test(timeout = 20000) public void testWithoutGrouping1() throws InterruptedException { InputReadyTracker inputReadyTracker = new InputReadyTracker(); @@ -64,7 +63,7 @@ public void testWithoutGrouping1() throws InterruptedException { assertTrue(input1.isReady); assertFalse(input2.isReady); assertEquals(input1, readyInput); - + startTime = System.nanoTime(); setDelayedInputReady(input2); assertFalse(inputReadyTracker.waitForAllInputsReady(requestList, 0)); @@ -89,12 +88,12 @@ public void testWithoutGrouping2() throws InterruptedException { List requestList; long startTime = 0l; long readyTime = 0l; - + requestList = new ArrayList(); requestList.add(input1); requestList.add(input2); requestList.add(input3); - + startTime = System.nanoTime(); setDelayedInputReady(input2); Input readyInput = inputReadyTracker.waitForAnyInputReady(requestList); @@ -106,7 +105,7 @@ public void testWithoutGrouping2() throws InterruptedException { assertTrue(readyTime >= startTime + SLEEP_TIME); assertFalse(input1.isReady); assertFalse(input3.isReady); - + requestList = new ArrayList(); requestList.add(input1); requestList.add(input3); @@ -121,7 +120,7 @@ public void testWithoutGrouping2() throws InterruptedException { assertTrue(readyTime >= startTime + SLEEP_TIME); assertTrue(input2.isReady); assertFalse(input3.isReady); - + requestList = new ArrayList(); requestList.add(input3); startTime = System.nanoTime(); @@ -145,16 +144,14 @@ public void testGrouped() throws InterruptedException { ImmediatelyReadyInputForTest input1 = new ImmediatelyReadyInputForTest(inputReadyTracker); ControlledReadyInputForTest input2 = new ControlledReadyInputForTest(inputReadyTracker); - + ImmediatelyReadyInputForTest input3 = new ImmediatelyReadyInputForTest(inputReadyTracker); ControlledReadyInputForTest input4 = new ControlledReadyInputForTest(inputReadyTracker); - - List group1Inputs = new ArrayList(); group1Inputs.add(input1); group1Inputs.add(input2); - + List group2Inputs = new ArrayList(); group2Inputs.add(input3); group2Inputs.add(input4); @@ -185,12 +182,10 @@ public void testGrouped() throws InterruptedException { assertTrue(input1.isReady); assertFalse(input2.isReady); assertEquals(group1, readyInput); - - + requestList = new ArrayList(); requestList.add(group2); - - + startTime = System.nanoTime(); setDelayedInputReady(input4); inputReadyTracker.waitForAllInputsReady(requestList); @@ -201,9 +196,8 @@ public void testGrouped() throws InterruptedException { assertTrue(input3.isReady); assertTrue(input4.isReady); assertTrue(readyTime >= startTime + SLEEP_TIME); - } - + private long setDelayedInputReady(final ControlledReadyInputForTest input) { long startTime = System.nanoTime(); new Thread() { @@ -222,7 +216,7 @@ public void run() { private static class ImmediatelyReadyInputForTest extends AbstractLogicalInput { private volatile boolean isReady = false; - + ImmediatelyReadyInputForTest(InputReadyTracker inputReadyTracker) { super(null, 0); isReady = true; @@ -252,12 +246,12 @@ public List close() throws Exception { return null; } } - + private static class ControlledReadyInputForTest extends AbstractLogicalInput { private volatile boolean isReady = false; private InputReadyTracker inputReadyTracker; - + ControlledReadyInputForTest(InputReadyTracker inputReadyTracker) { super(null, 0); this.inputReadyTracker = inputReadyTracker; @@ -269,7 +263,7 @@ public List initialize() throws Exception { } @Override - public void start() throws Exception { + public void start() throws Exception { } @Override @@ -286,7 +280,7 @@ public List close() throws Exception { return null; } - // Used by the test to control when this input will be ready + // Used by the test to control when this input will be ready public void setInputIsReady() { isReady = true; inputReadyTracker.setInputIsReady(this); diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestLogicalIOProcessorRuntimeTask.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestLogicalIOProcessorRuntimeTask.java index 1524cac357..191915ca30 100644 --- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestLogicalIOProcessorRuntimeTask.java +++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestLogicalIOProcessorRuntimeTask.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -79,7 +79,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Multimap; - public class TestLogicalIOProcessorRuntimeTask { @Test(timeout = 5000) @@ -126,7 +125,7 @@ public void testAutoStart() throws Exception { assertEquals(30, lio1.getProcessorContext().getVertexParallelism()); assertEquals(30, lio1.getInputContexts().iterator().next().getVertexParallelism()); assertEquals(30, lio1.getOutputContexts().iterator().next().getVertexParallelism()); - } catch(Exception e) { + } catch (Exception e) { fail(); sharedExecutor.shutdownNow(); } finally { @@ -154,13 +153,12 @@ public void testAutoStart() throws Exception { assertEquals(10, lio2.getProcessorContext().getVertexParallelism()); assertEquals(10, lio2.getInputContexts().iterator().next().getVertexParallelism()); assertEquals(10, lio2.getOutputContexts().iterator().next().getVertexParallelism()); - } catch(Exception e) { + } catch (Exception e) { fail(); } finally { cleanupAndTest(lio2); sharedExecutor.shutdownNow(); } - } @Test @@ -193,7 +191,7 @@ public void testEventsCantBeSentInCleanup() throws Exception { runner.call(); // We verify that no events were sent - verify(umbilical, only()).addEvents(Collections. emptyList()); + verify(umbilical, only()).addEvents(Collections.emptyList()); } /** @@ -288,8 +286,8 @@ private void cleanupAndTest(LogicalIOProcessorRuntimeTask lio) throws Interrupte } private TaskSpec createTaskSpec(TezTaskAttemptID taskAttemptID, - String dagName, String vertexName, int parallelism, - String processorClassname, String outputClassName) { + String dagName, String vertexName, int parallelism, + String processorClassname, String outputClassName) { ProcessorDescriptor processorDesc = createProcessorDescriptor(processorClassname); TaskSpec taskSpec = new TaskSpec(taskAttemptID, dagName, vertexName, parallelism, processorDesc, @@ -331,24 +329,25 @@ private TezDAGID createTezDagId() { private static class CleanupLogicalIOProcessorRuntimeTask extends LogicalIOProcessorRuntimeTask { CleanupLogicalIOProcessorRuntimeTask(TaskSpec taskSpec, - int appAttemptNumber, Configuration tezConf, String[] localDirs, - TezUmbilical tezUmbilical, - Map serviceConsumerMetadata, - Map envMap, Multimap startedInputsMap, - ObjectRegistry objectRegistry, String pid, - org.apache.tez.runtime.api.ExecutionContext ExecutionContext, - long memAvailable, boolean updateSysCounters, HadoopShim hadoopShim, - TezExecutors sharedExecutor) throws IOException { + int appAttemptNumber, Configuration tezConf, String[] localDirs, + TezUmbilical tezUmbilical, + Map serviceConsumerMetadata, + Map envMap, Multimap startedInputsMap, + ObjectRegistry objectRegistry, String pid, + org.apache.tez.runtime.api.ExecutionContext ExecutionContext, + long memAvailable, boolean updateSysCounters, HadoopShim hadoopShim, + TezExecutors sharedExecutor) throws IOException { super(taskSpec, appAttemptNumber, tezConf, localDirs, tezUmbilical, serviceConsumerMetadata, envMap, startedInputsMap, objectRegistry, pid, ExecutionContext, memAvailable, updateSysCounters, hadoopShim, sharedExecutor); } - @Override public void cleanup() throws InterruptedException { + @Override + public void cleanup() throws InterruptedException { getOutputContexts().forEach(context -> context.sendEvents(Arrays.asList( - CompositeDataMovementEvent.create(0, 0, null) + CompositeDataMovementEvent.create(0, 0, null) ))); } } @@ -379,7 +378,6 @@ public void handleEvents(List processorEvents) { @Override public void close() throws Exception { } - } public static class RunExceptionProcessor @@ -390,7 +388,7 @@ public RunExceptionProcessor(ProcessorContext context) { } public void run(Map inputs, - Map outputs) + Map outputs) throws Exception { // This exception is thrown in purpose because we want to test this throw new RuntimeException(); @@ -413,7 +411,6 @@ public CloseExceptionProcessor(ProcessorContext context) { public void close() throws Exception { throw new RuntimeException(); } - } public static class TestInput extends AbstractLogicalInput { @@ -452,7 +449,6 @@ public void handleEvents(List inputEvents) throws Exception { public List close() throws Exception { return null; } - } public static class TestOutput extends AbstractLogicalOutput { @@ -464,7 +460,6 @@ public TestOutput(OutputContext outputContext, int numPhysicalOutputs) { super(outputContext, numPhysicalOutputs); } - @Override public List initialize() throws Exception { getContext().requestInitialMemory(0, null); diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/api/impl/TestProcessorContext.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/api/impl/TestProcessorContext.java index bf4fdf61d3..fe00490e17 100644 --- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/api/impl/TestProcessorContext.java +++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/api/impl/TestProcessorContext.java @@ -44,9 +44,9 @@ public class TestProcessorContext { - @Test (timeout = 5000) + @Test(timeout = 5000) public void testDagNumber() throws IOException { - String[] localDirs = new String[] {"dummyLocalDir"}; + String[] localDirs = new String[]{"dummyLocalDir"}; int appAttemptNumber = 1; TezUmbilical tezUmbilical = mock(TezUmbilical.class); String dagName = "DAG_NAME"; @@ -107,9 +107,9 @@ public void testDagNumber() throws IOException { assertEquals(vertexId.getId(), procContext.getTaskVertexIndex()); assertTrue(Arrays.equals(localDirs, procContext.getWorkDirs())); - // test auto call of notifyProgress - procContext.setProgress(0.1f); - verify(mockTask, times(1)).notifyProgressInvocation(); - sharedExecutor.shutdown(); + // test auto call of notifyProgress + procContext.setProgress(0.1f); + verify(mockTask, times(1)).notifyProgressInvocation(); + sharedExecutor.shutdown(); } } diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/api/impl/TestTaskSpec.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/api/impl/TestTaskSpec.java index dfe19f8cd9..c4b2c754d6 100644 --- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/api/impl/TestTaskSpec.java +++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/api/impl/TestTaskSpec.java @@ -43,14 +43,14 @@ public class TestTaskSpec { - @Test (timeout = 5000) + @Test(timeout = 5000) public void testSerDe() throws IOException { ByteBuffer payload = null; ProcessorDescriptor procDesc = ProcessorDescriptor.create("proc").setUserPayload( UserPayload.create(payload)).setHistoryText("historyText"); List inputSpecs = new ArrayList<>(); - InputSpec inputSpec = new InputSpec("src1", InputDescriptor.create("inputClass"),10); + InputSpec inputSpec = new InputSpec("src1", InputDescriptor.create("inputClass"), 10); inputSpecs.add(inputSpec); List outputSpecs = new ArrayList<>(); OutputSpec outputSpec = new OutputSpec("dest1", OutputDescriptor.create("outputClass"), 999); @@ -87,5 +87,4 @@ public void testSerDe() throws IOException { Assert.assertEquals(taskConf.get("foo"), deSerTaskSpec.getTaskConf().get("foo")); } - } diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/api/impl/TestTezEvent.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/api/impl/TestTezEvent.java index b39c4edf6e..f750d98361 100644 --- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/api/impl/TestTezEvent.java +++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/api/impl/TestTezEvent.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -98,7 +98,6 @@ public void testSerialization() throws IOException { actual2 = deserializeEvents(dis); assertEventEquals(events, actual1); assertEventEquals(events, actual2); - } private void serializeEvents(ArrayList events, DataOutput out) throws IOException { @@ -148,5 +147,4 @@ private void assertEventEquals(ArrayList expectedList, ArrayList + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -40,7 +40,6 @@ private void testCRUD(ObjectRegistry objectRegistry) { Assert.assertEquals(two_2, objectRegistry.get("two")); Assert.assertTrue(objectRegistry.delete("one")); Assert.assertFalse(objectRegistry.delete("one")); - } @Test(timeout = 5000) @@ -59,12 +58,12 @@ public void testClearCache() { objectRegistry.cacheForVertex(one, one); objectRegistry.cacheForDAG(two, two); - ((ObjectRegistryImpl)objectRegistry).clearCache(ObjectRegistryImpl.ObjectLifeCycle.VERTEX); + ((ObjectRegistryImpl) objectRegistry).clearCache(ObjectRegistryImpl.ObjectLifeCycle.VERTEX); Assert.assertNull(objectRegistry.get(one)); Assert.assertNotNull(objectRegistry.get(two)); objectRegistry.cacheForVertex(one, one); - ((ObjectRegistryImpl)objectRegistry).clearCache(ObjectRegistryImpl.ObjectLifeCycle.DAG); + ((ObjectRegistryImpl) objectRegistry).clearCache(ObjectRegistryImpl.ObjectLifeCycle.DAG); Assert.assertNotNull(objectRegistry.get(one)); Assert.assertNull(objectRegistry.get(two)); } diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/resources/TestMemoryDistributor.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/resources/TestMemoryDistributor.java index 7fbd87f08a..bee7ff4376 100644 --- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/resources/TestMemoryDistributor.java +++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/common/resources/TestMemoryDistributor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -39,18 +39,18 @@ public class TestMemoryDistributor { protected Configuration conf = new Configuration(); - + @Before public void setup() { conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ENABLED, true); conf.set(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ALLOCATOR_CLASS, ScalingAllocator.class.getName()); } - + @Test(timeout = 5000) public void testScalingNoProcessor() throws TezException { MemoryDistributor dist = new MemoryDistributor(2, 1, conf); - + dist.setJvmMemory(10000l); // First request @@ -58,21 +58,21 @@ public void testScalingNoProcessor() throws TezException { InputContext e1InputContext1 = createTestInputContext(); InputDescriptor e1InDesc1 = createTestInputDescriptor(); dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1); - + // Second request MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest(); InputContext e2InputContext2 = createTestInputContext(); InputDescriptor e2InDesc2 = createTestInputDescriptor(); dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2); - + // Third request - output MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest(); OutputContext e3OutputContext1 = createTestOutputContext(); OutputDescriptor e3OutDesc2 = createTestOutputDescriptor(); dist.requestMemory(5000, e3Callback, e3OutputContext1, e3OutDesc2); - + dist.makeInitialAllocations(); - + // Total available: 70% of 10K = 7000 // 3 requests - 10K, 10K, 5K // Scale down to - 2800, 2800, 1400 @@ -80,12 +80,12 @@ public void testScalingNoProcessor() throws TezException { assertEquals(2800, e2Callback.assigned); assertEquals(1400, e3Callback.assigned); } - + @Test(timeout = 5000) public void testScalingNoProcessor2() throws TezException { // Real world values MemoryDistributor dist = new MemoryDistributor(2, 0, conf); - + dist.setJvmMemory(209715200l); // First request @@ -93,23 +93,23 @@ public void testScalingNoProcessor2() throws TezException { InputContext e1InputContext1 = createTestInputContext(); InputDescriptor e1InDesc1 = createTestInputDescriptor(); dist.requestMemory(104857600l, e1Callback, e1InputContext1, e1InDesc1); - + // Second request MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest(); InputContext e2InputContext2 = createTestInputContext(); InputDescriptor e2InDesc2 = createTestInputDescriptor(); dist.requestMemory(157286400l, e2Callback, e2InputContext2, e2InDesc2); - + dist.makeInitialAllocations(); assertEquals(58720256l, e1Callback.assigned); assertEquals(88080384l, e2Callback.assigned); } - + @Test(timeout = 5000) public void testScalingProcessor() throws TezException { MemoryDistributor dist = new MemoryDistributor(2, 1, conf); - + dist.setJvmMemory(10000l); // First request @@ -117,28 +117,27 @@ public void testScalingProcessor() throws TezException { InputContext e1InputContext1 = createTestInputContext(); InputDescriptor e1InDesc1 = createTestInputDescriptor(); dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1); - + // Second request MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest(); InputContext e2InputContext2 = createTestInputContext(); InputDescriptor e2InDesc2 = createTestInputDescriptor(); dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2); - + // Third request - output MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest(); OutputContext e3OutputContext1 = createTestOutputContext(); OutputDescriptor e3OutDesc1 = createTestOutputDescriptor(); dist.requestMemory(5000, e3Callback, e3OutputContext1, e3OutDesc1); - + // Fourth request - processor MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest(); ProcessorContext e4ProcessorContext1 = createTestProcessortContext(); ProcessorDescriptor e4ProcessorDesc1 = createTestProcessorDescriptor(); dist.requestMemory(5000, e4Callback, e4ProcessorContext1, e4ProcessorDesc1); - - + dist.makeInitialAllocations(); - + // Total available: 70% of 10K = 7000 // 4 requests - 10K, 10K, 5K, 5K // Scale down to - 2333.33, 2333.33, 1166.66, 1166.66 @@ -147,14 +146,14 @@ public void testScalingProcessor() throws TezException { assertTrue(e3Callback.assigned >= 1166 && e3Callback.assigned <= 1167); assertTrue(e4Callback.assigned >= 1166 && e4Callback.assigned <= 1167); } - + @Test(timeout = 5000) public void testScalingDisabled() throws TezException { // Real world values Configuration conf = new Configuration(this.conf); conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ENABLED, false); MemoryDistributor dist = new MemoryDistributor(2, 0, conf); - + dist.setJvmMemory(207093760l); // First request @@ -162,25 +161,25 @@ public void testScalingDisabled() throws TezException { InputContext e1InputContext1 = createTestInputContext(); InputDescriptor e1InDesc1 = createTestInputDescriptor(); dist.requestMemory(104857600l, e1Callback, e1InputContext1, e1InDesc1); - + // Second request MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest(); InputContext e2InputContext2 = createTestInputContext(); InputDescriptor e2InDesc2 = createTestInputDescriptor(); dist.requestMemory(144965632l, e2Callback, e2InputContext2, e2InDesc2); - + dist.makeInitialAllocations(); assertEquals(104857600l, e1Callback.assigned); assertEquals(144965632l, e2Callback.assigned); } - + @Test(timeout = 5000) public void testReserveFractionConfigured() throws TezException { Configuration conf = new Configuration(this.conf); conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_RESERVE_FRACTION, 0.5d); MemoryDistributor dist = new MemoryDistributor(2, 1, conf); - + dist.setJvmMemory(10000l); // First request @@ -188,21 +187,21 @@ public void testReserveFractionConfigured() throws TezException { InputContext e1InputContext1 = createTestInputContext(); InputDescriptor e1InDesc1 = createTestInputDescriptor(); dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1); - + // Second request MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest(); InputContext e2InputContext2 = createTestInputContext(); InputDescriptor e2InDesc2 = createTestInputDescriptor(); dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2); - + // Third request - output MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest(); OutputContext e3OutputContext1 = createTestOutputContext(); OutputDescriptor e3OutDesc2 = createTestOutputDescriptor(); dist.requestMemory(5000, e3Callback, e3OutputContext1, e3OutDesc2); - + dist.makeInitialAllocations(); - + // Total available: 50% of 10K = 7000 // 3 requests - 10K, 10K, 5K // Scale down to - 2000, 2000, 1000 @@ -210,8 +209,7 @@ public void testReserveFractionConfigured() throws TezException { assertEquals(2000, e2Callback.assigned); assertEquals(1000, e3Callback.assigned); } - - + private static class MemoryUpdateCallbackForTest extends MemoryUpdateCallback { long assigned = -1000; @@ -246,18 +244,17 @@ protected InputContext createTestInputContext() { doReturn("task").when(context).getTaskVertexName(); return context; } - + protected OutputContext createTestOutputContext() { OutputContext context = mock(OutputContext.class); doReturn("output").when(context).getDestinationVertexName(); doReturn("task").when(context).getTaskVertexName(); return context; } - + protected ProcessorContext createTestProcessortContext() { ProcessorContext context = mock(ProcessorContext.class); doReturn("task").when(context).getTaskVertexName(); return context; } - } diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TaskExecutionTestHelpers.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TaskExecutionTestHelpers.java index 626d178cec..cc2250c568 100644 --- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TaskExecutionTestHelpers.java +++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TaskExecutionTestHelpers.java @@ -443,7 +443,6 @@ public void verifyTaskKilledEvent(String diagStart, String diagContains) { } finally { umbilicalLock.unlock(); } - } public void verifyTaskSuccessEvent() { diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestContainerExecution.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestContainerExecution.java index c3c4705c15..f35d77bc07 100644 --- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestContainerExecution.java +++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestContainerExecution.java @@ -52,7 +52,6 @@ public void testGetTaskShouldDie() throws InterruptedException, ExecutionExcepti getTaskFuture.get(); assertEquals(1, umbilical.getTaskInvocations); - } finally { executor.shutdownNow(); } diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java index 810c3ac82b..1c81c886c0 100644 --- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java +++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskExecution2.java @@ -198,7 +198,6 @@ public void testMultipleSuccessfulTasks() throws IOException, InterruptedExcepti } } - // test task failed due to exception in Processor @Test(timeout = 5000) public void testFailedTaskTezException() throws IOException, InterruptedException, TezException, @@ -236,7 +235,6 @@ public void testFailedTaskTezException() throws IOException, InterruptedExceptio } } - // Test task failed due to Processor class not found @Test(timeout = 5000) public void testFailedTask2() throws IOException, InterruptedException, TezException, @@ -298,7 +296,6 @@ public void testFailedTaskIOException() throws IOException, InterruptedException TaskRunner2Result result = taskRunnerFuture.get(); verifyTaskRunnerResult(result, EndReason.TASK_ERROR, createProcessorIOException(), false, TaskFailureType.NON_FATAL); - assertNull(taskReporter.currentCallable); umbilical.verifyTaskFailedEvent( FAILURE_START_STRING, @@ -325,7 +322,7 @@ public void testIgnoreErrorsDuringFailure() throws IOException, InterruptedExcep umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest(); TaskReporter taskReporter = new TaskReporter(umbilical, 100, 1000, 100, new AtomicLong(0), - createContainerId(appId).toString()) { + createContainerId(appId).toString()) { @Override protected boolean isShuttingDown() { return true; @@ -415,7 +412,6 @@ public void testHeartbeatShouldDie() throws IOException, InterruptedException, T TaskRunner2Result result = taskRunnerFuture.get(); verifyTaskRunnerResult(result, EndReason.CONTAINER_STOP_REQUESTED, null, true, null); - TestProcessor.awaitCompletion(); assertTrue(TestProcessor.wasInterrupted()); assertNull(taskReporter.currentCallable); @@ -656,7 +652,7 @@ public void testKilledAfterComplete() throws IOException, InterruptedException, private void verifySysCounters(TezCounters tezCounters, int minTaskCounterCount, int minFsCounterCount) { Preconditions.checkArgument((minTaskCounterCount > 0 && minFsCounterCount > 0) || - (minTaskCounterCount <= 0 && minFsCounterCount <= 0), + (minTaskCounterCount <= 0 && minFsCounterCount <= 0), "Both targetCounter counts should be postitive or negative. A mix is not expected"); int numTaskCounters = 0; @@ -708,13 +704,11 @@ private void verifyTaskRunnerResult(TaskRunner2Result taskRunner2Result, if (expectedExceptionMessage != null) { assertTrue(cause.getMessage().contains(expectedExceptionMessage)); } - } assertEquals(taskFailureType, taskRunner2Result.getTaskFailureType()); assertEquals(wasShutdownRequested, taskRunner2Result.isContainerShutdownRequested()); } - private static class TaskRunnerCallable2ForTest implements Callable { private final TezTaskRunner2 taskRunner; @@ -734,7 +728,6 @@ private TezTaskRunner2 createTaskRunner(ApplicationId appId, ListeningExecutorService executor, byte[] processorConf) throws IOException { return createTaskRunner(appId, umbilical, taskReporter, executor, processorConf, true); - } private TezTaskRunner2 createTaskRunner(ApplicationId appId, @@ -827,7 +820,6 @@ executionContext, memAvailable, updateSysCounters, new DefaultHadoopShim(), sharedExecutor); } - @Override @VisibleForTesting void processCallableResult(TaskRunner2Callable.TaskRunner2CallableResult executionResult) { @@ -852,5 +844,4 @@ void awaitCallableCompletion() throws InterruptedException { } } } - } diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskReporter.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskReporter.java index 147d17655b..0163822cd8 100644 --- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskReporter.java +++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTaskReporter.java @@ -112,9 +112,8 @@ public Object answer(InvocationOnMock invocation) throws Throwable { } finally { executor.shutdownNow(); } - } - + @Test(timeout = 10000) public void testEventThrottling() throws Exception { TezTaskAttemptID mockTaskAttemptId = mock(TezTaskAttemptID.class); @@ -152,7 +151,7 @@ public void testEventThrottling() throws Exception { Assert.assertEquals(1, req.getMaxEvents()); } - @Test (timeout=5000) + @Test(timeout = 5000) public void testStatusUpdateAfterInitializationAndCounterFlag() { TezTaskAttemptID mockTaskAttemptId = mock(TezTaskAttemptID.class); LogicalIOProcessorRuntimeTask mockTask = mock(LogicalIOProcessorRuntimeTask.class); @@ -161,20 +160,20 @@ public void testStatusUpdateAfterInitializationAndCounterFlag() { boolean progressNotified = false; doReturn(progressNotified).when(mockTask).getAndClearProgressNotification(); TezTaskUmbilicalProtocol mockUmbilical = mock(TezTaskUmbilicalProtocol.class); - + float progress = 0.5f; TaskStatistics stats = new TaskStatistics(); TezCounters counters = new TezCounters(); doReturn(progress).when(mockTask).getProgress(); doReturn(stats).when(mockTask).getTaskStatistics(); doReturn(counters).when(mockTask).getCounters(); - + // Setup the sleep time to be way higher than the test timeout TaskReporter.HeartbeatCallable heartbeatCallable = new TaskReporter.HeartbeatCallable(mockTask, mockUmbilical, 100000, 100000, 5, new AtomicLong(0), "containerIdStr"); - + // task not initialized - nothing obtained from task doReturn(false).when(mockTask).hasInitialized(); TaskStatusUpdateEvent event = heartbeatCallable.getStatusUpdateEvent(true); @@ -215,7 +214,6 @@ public void testStatusUpdateAfterInitializationAndCounterFlag() { Assert.assertEquals(progressNotified, event.getProgressNotified()); Assert.assertEquals(counters, event.getCounters()); Assert.assertEquals(stats, event.getStatistics()); - } private List createEvents(int numEvents) { diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTezTaskRunner2.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTezTaskRunner2.java index 6876df93ec..e6384c7c4f 100644 --- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTezTaskRunner2.java +++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/task/TestTezTaskRunner2.java @@ -35,10 +35,9 @@ import org.junit.Assert; import org.junit.Test; - public class TestTezTaskRunner2 { - @Test (timeout = 5000) + @Test(timeout = 5000) public void testTaskConfUsage() throws Exception { Configuration conf = new Configuration(false); conf.set("global", "global1"); @@ -62,6 +61,4 @@ public void testTaskConfUsage() throws Exception { Assert.assertEquals("task1", taskRunner2.task.getTaskConf().get("task")); sharedExecutor.shutdownNow(); } - - } diff --git a/tez-runtime-library/src/main/java/org/apache/hadoop/io/FileChunk.java b/tez-runtime-library/src/main/java/org/apache/hadoop/io/FileChunk.java index 714bbcd484..399a090556 100644 --- a/tez-runtime-library/src/main/java/org/apache/hadoop/io/FileChunk.java +++ b/tez-runtime-library/src/main/java/org/apache/hadoop/io/FileChunk.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -59,7 +59,7 @@ public boolean equals(Object o) { return false; } - FileChunk that = (FileChunk)o; + FileChunk that = (FileChunk) o; return path.equals(that.path) && (offset == that.offset) && (length == that.length); } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/edgemanager/SilentEdgeManager.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/edgemanager/SilentEdgeManager.java index db6bb5affe..e334d75e16 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/edgemanager/SilentEdgeManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/edgemanager/SilentEdgeManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/DestinationTaskInputsProperty.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/DestinationTaskInputsProperty.java index bb23f19699..942aaf5e6a 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/DestinationTaskInputsProperty.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/DestinationTaskInputsProperty.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.library.vertexmanager; @@ -28,30 +28,37 @@ class DestinationTaskInputsProperty { private final int numOfPartitions; private final int firstSourceTaskIndex; private final int numOfSourceTasks; + public DestinationTaskInputsProperty(int firstPartitionId, - int numOfPartitions, int firstSourceTaskIndex, int numOfSourceTasks) { + int numOfPartitions, int firstSourceTaskIndex, int numOfSourceTasks) { this.firstPartitionId = firstPartitionId; this.numOfPartitions = numOfPartitions; this.firstSourceTaskIndex = firstSourceTaskIndex; this.numOfSourceTasks = numOfSourceTasks; } + public int getFirstPartitionId() { return firstPartitionId; } + public int getNumOfPartitions() { return numOfPartitions; } + public int getFirstSourceTaskIndex() { return firstSourceTaskIndex; } + public int getNumOfSourceTasks() { return numOfSourceTasks; } + public boolean isSourceTaskInRange(int sourceTaskIndex) { return firstSourceTaskIndex <= sourceTaskIndex && sourceTaskIndex < firstSourceTaskIndex + numOfSourceTasks; } + public boolean isPartitionInRange(int partitionId) { return firstPartitionId <= partitionId && partitionId < firstPartitionId + numOfPartitions; @@ -89,4 +96,3 @@ public String toString() { " ,numOfSourceTasks = " + numOfSourceTasks; } } - diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairEdgeConfiguration.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairEdgeConfiguration.java index 846e0a3745..ef9f5a81e4 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairEdgeConfiguration.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairEdgeConfiguration.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.library.vertexmanager; @@ -30,7 +30,6 @@ import java.util.HashMap; import java.util.Map.Entry; - /** * Handles edge configuration serialization and de-serialization between * {@link FairShuffleVertexManager} and {@link FairShuffleEdgeManager}. @@ -41,7 +40,7 @@ class FairEdgeConfiguration { destinationInputsProperties; public FairEdgeConfiguration(int numBuckets, - HashMap routingTable) { + HashMap routingTable) { this.destinationInputsProperties = routingTable; this.numBuckets = numBuckets; } @@ -58,9 +57,9 @@ private FairShuffleEdgeManagerConfigPayloadProto getConfigPayload() { taskBuilder. setDestinationTaskIndex(entry.getKey()). setPartitions(newRange(entry.getValue().getFirstPartitionId(), - entry.getValue().getNumOfPartitions())). + entry.getValue().getNumOfPartitions())). setSourceTasks(newRange(entry.getValue(). - getFirstSourceTaskIndex(), entry.getValue().getNumOfSourceTasks())); + getFirstSourceTaskIndex(), entry.getValue().getNumOfSourceTasks())); builder.addDestinationTaskProps(taskBuilder.build()); } } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairShuffleEdgeManager.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairShuffleEdgeManager.java index 2336e15723..5e8d545898 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairShuffleEdgeManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairShuffleEdgeManager.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.library.vertexmanager; @@ -59,7 +59,7 @@ public int getNumSourceTaskPhysicalOutputs(int sourceTaskIndex) { @Override public int getNumDestinationConsumerTasks(int sourceTaskIndex) { int numTasks = 0; - for(DestinationTaskInputsProperty entry: mapping.values()) { + for (DestinationTaskInputsProperty entry : mapping.values()) { if (entry.isSourceTaskInRange(sourceTaskIndex)) { numTasks++; } @@ -87,7 +87,7 @@ public void initialize() { @Override public int routeInputErrorEventToSource(int destinationTaskIndex, - int destinationFailedInputIndex) { + int destinationFailedInputIndex) { return mapping.get(destinationTaskIndex).getSourceTaskIndex( destinationFailedInputIndex); } @@ -122,8 +122,8 @@ private int[] getRange(int startValue, int count) { @Override public @Nullable CompositeEventRouteMetadata - routeCompositeDataMovementEventToDestination(int sourceTaskIndex, - int destinationTaskIndex) { + routeCompositeDataMovementEventToDestination(int sourceTaskIndex, + int destinationTaskIndex) { DestinationTaskInputsProperty property = mapping.get(destinationTaskIndex); int firstPhysicalInputIndex = property.getFirstPhysicalInputIndex(sourceTaskIndex); @@ -150,4 +150,3 @@ public EventRouteMetadata routeInputSourceTaskFailedEventToDestination( } } } - diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairShuffleVertexManager.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairShuffleVertexManager.java index af4e5b8b26..b913f23546 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairShuffleVertexManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/FairShuffleVertexManager.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.library.vertexmanager; @@ -47,7 +47,6 @@ import java.util.Map; import java.util.NoSuchElementException; - /** * Fair routing based on partition size distribution to achieve optimal * input size for any destination task thus reduce data skewness. @@ -84,7 +83,7 @@ public class FairShuffleVertexManager extends ShuffleVertexManagerBase { "tez.fair-shuffle-vertex-manager.enable.auto-parallel"; public static final String TEZ_FAIR_SHUFFLE_VERTEX_MANAGER_ENABLE_AUTO_PARALLEL_DEFAULT = - FairRoutingType.NONE.getType(); + FairRoutingType.NONE.getType(); /** * In case of a ScatterGather connection, the fraction of source tasks which @@ -170,22 +169,22 @@ static class FairSourceVertexInfo extends SourceVertexInfo { destinationInputsProperties = new HashMap<>(); FairSourceVertexInfo(final EdgeProperty edgeProperty, - int totalTasksToSchedule) { + int totalTasksToSchedule) { super(edgeProperty, totalTasksToSchedule); } + public HashMap - getDestinationInputsProperties() { + getDestinationInputsProperties() { return destinationInputsProperties; } } @Override SourceVertexInfo createSourceVertexInfo(EdgeProperty edgeProperty, - int numTasks) { + int numTasks) { return new FairSourceVertexInfo(edgeProperty, numTasks); } - FairShuffleVertexManagerConfig mgrConfig; public FairShuffleVertexManager(VertexManagerPluginContext context) { @@ -225,8 +224,8 @@ public long[] estimatePartitionSize() { // distributed. if (numOfPartitions > 0) { long estimatedPerPartitionSize = - getExpectedTotalBipartiteSourceTasksOutputSize().divide( - BigInteger.valueOf(numOfPartitions)).longValue(); + getExpectedTotalBipartiteSourceTasksOutputSize().divide( + BigInteger.valueOf(numOfPartitions)).longValue(); for (int i = 0; i < numOfPartitions; i++) { estimatedPartitionOutputSize[i] = estimatedPerPartitionSize; } @@ -292,8 +291,9 @@ private class PartitionsGroupingCalculator // numOfBaseDestinationTasks == 1, numOfBaseSourceTasks == 2. private int numOfBaseSourceTasks = 0; private int numOfBaseDestinationTasks = 0; + public PartitionsGroupingCalculator(long[] estimatedPartitionOutputSize, - FairSourceVertexInfo sourceVertexInfo) { + FairSourceVertexInfo sourceVertexInfo) { this.estimatedPartitionOutputSize = estimatedPartitionOutputSize; this.sourceVertexInfo = sourceVertexInfo; } @@ -420,7 +420,7 @@ public void compute() { continue; } Iterator it = iterator(); - while(it.hasNext()) { + while (it.hasNext()) { DestinationTaskInputsProperty property = it.next(); sourceVertexInfo.getDestinationInputsProperties().put( destinationIndex, property); @@ -438,12 +438,12 @@ public ReconfigVertexParams computeRouting() { int finalTaskParallelism = 0; long[] estimatedPartitionOutputSize = estimatePartitionSize(); for (Map.Entry vInfo : getBipartiteInfo()) { - FairSourceVertexInfo info = (FairSourceVertexInfo)vInfo.getValue(); + FairSourceVertexInfo info = (FairSourceVertexInfo) vInfo.getValue(); computeParallelism(estimatedPartitionOutputSize, info); if (finalTaskParallelism != 0) { Preconditions.checkState( finalTaskParallelism == info.getDestinationInputsProperties().size(), - "the parallelism shall be the same for source vertices"); + "the parallelism shall be the same for source vertices"); } finalTaskParallelism = info.getDestinationInputsProperties().size(); @@ -470,7 +470,7 @@ void processPendingTasks() { } private void computeParallelism(long[] estimatedPartitionOutputSize, - FairSourceVertexInfo sourceVertexInfo) { + FairSourceVertexInfo sourceVertexInfo) { PartitionsGroupingCalculator calculator = new PartitionsGroupingCalculator( estimatedPartitionOutputSize, sourceVertexInfo); calculator.compute(); @@ -495,7 +495,7 @@ List getTasksToSchedule( if (completedSourceAttempt != null) { srcTaskId = completedSourceAttempt.getTaskIdentifier().getIdentifier(); String srcVertexName = completedSourceAttempt.getTaskIdentifier().getVertexIdentifier().getName(); - srcInfo = (FairSourceVertexInfo)getSourceVertexInfo(srcVertexName); + srcInfo = (FairSourceVertexInfo) getSourceVertexInfo(srcVertexName); } while (it.hasNext() && numTasksToSchedule > 0) { Integer taskIndex = it.next().getIndex(); @@ -524,14 +524,16 @@ List getTasksToSchedule( static class FairShuffleVertexManagerConfig extends ShuffleVertexManagerBaseConfig { final FairRoutingType fairRoutingType; + public FairShuffleVertexManagerConfig(final boolean enableAutoParallelism, - final long desiredTaskInputDataSize, final float slowStartMinFraction, - final float slowStartMaxFraction, final FairRoutingType fairRoutingType) { + final long desiredTaskInputDataSize, final float slowStartMinFraction, + final float slowStartMaxFraction, final FairRoutingType fairRoutingType) { super(enableAutoParallelism, desiredTaskInputDataSize, slowStartMinFraction, slowStartMaxFraction); this.fairRoutingType = fairRoutingType; LOG.info("fairRoutingType {}", this.fairRoutingType); } + FairRoutingType getFairRoutingType() { return fairRoutingType; } @@ -555,7 +557,7 @@ ShuffleVertexManagerBaseConfig initConfiguration() { conf.getFloat( TEZ_FAIR_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION, Math.max(slowStartMinFraction, - TEZ_FAIR_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION_DEFAULT)), + TEZ_FAIR_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION_DEFAULT)), fairRoutingType); return mgrConfig; } @@ -574,7 +576,7 @@ ShuffleVertexManagerBaseConfig initConfiguration() { * @return {@link FairShuffleVertexManagerConfigBuilder} */ public static FairShuffleVertexManagerConfigBuilder - createConfigBuilder(@Nullable Configuration conf) { + createConfigBuilder(@Nullable Configuration conf) { return new FairShuffleVertexManagerConfigBuilder(conf); } @@ -600,14 +602,14 @@ public FairShuffleVertexManagerConfigBuilder setAutoParallelism( } public FairShuffleVertexManagerConfigBuilder - setSlowStartMinSrcCompletionFraction(float minFraction) { + setSlowStartMinSrcCompletionFraction(float minFraction) { conf.setFloat(TEZ_FAIR_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, minFraction); return this; } public FairShuffleVertexManagerConfigBuilder - setSlowStartMaxSrcCompletionFraction(float maxFraction) { + setSlowStartMaxSrcCompletionFraction(float maxFraction) { conf.setFloat(TEZ_FAIR_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION, maxFraction); return this; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/InputReadyVertexManager.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/InputReadyVertexManager.java index 33c4a99059..ba5fd1eaac 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/InputReadyVertexManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/InputReadyVertexManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -46,7 +46,7 @@ @Private public class InputReadyVertexManager extends VertexManagerPlugin { - private static final Logger LOG = + private static final Logger LOG = LoggerFactory.getLogger(InputReadyVertexManager.class); Map srcVertexInfo = Maps.newHashMap(); @@ -68,7 +68,7 @@ static class SourceVertexInfo { int numTasks; int numFinishedTasks; Boolean taskIsFinished[]; - + SourceVertexInfo(int numTasks, EdgeProperty edgeProperty) { this.numTasks = numTasks; this.numFinishedTasks = 0; @@ -76,7 +76,7 @@ static class SourceVertexInfo { this.taskIsFinished = new Boolean[numTasks]; } } - + private void configure() { Preconditions.checkState(!configured.get(), "Vertex: " + getContext().getVertexName()); int numManagedTasks = getContext().getVertexNumTasks(getContext().getVertexName()); @@ -92,42 +92,42 @@ private void configure() { String srcVertex = entry.getKey(); int numSrcTasks = getContext().getVertexNumTasks(srcVertex); switch (edgeProp.getDataMovementType()) { - case CUSTOM: - throw new TezUncheckedException("Cannot handle custom edge"); - case ONE_TO_ONE: - numOneToOneEdges++; - if (oneToOneSrcTaskCount == 0) { - oneToOneSrcTaskCount = numSrcTasks; - } else if (oneToOneSrcTaskCount != numSrcTasks) { + case CUSTOM: + throw new TezUncheckedException("Cannot handle custom edge"); + case ONE_TO_ONE: + numOneToOneEdges++; + if (oneToOneSrcTaskCount == 0) { + oneToOneSrcTaskCount = numSrcTasks; + } else if (oneToOneSrcTaskCount != numSrcTasks) { + throw new TezUncheckedException( + "All 1-1 source vertices must have identical concurrency"); + } + break; + case SCATTER_GATHER: + case BROADCAST: + break; + default: throw new TezUncheckedException( - "All 1-1 source vertices must have identical concurrency"); - } - break; - case SCATTER_GATHER: - case BROADCAST: - break; - default: - throw new TezUncheckedException( - "Unknown edge type: " + edgeProp.getDataMovementType()); + "Unknown edge type: " + edgeProp.getDataMovementType()); } srcVertexInfo.put(srcVertex, new SourceVertexInfo(numSrcTasks, edgeProp)); } - + if (numOneToOneEdges > 0) { Preconditions .checkState(oneToOneSrcTaskCount >= 0, "Vertex: " + getContext().getVertexName()); if (oneToOneSrcTaskCount != numManagedTasks) { numManagedTasks = oneToOneSrcTaskCount; // must change parallelism to make them the same - LOG.info("Update parallelism of vertex: " + getContext().getVertexName() + + LOG.info("Update parallelism of vertex: " + getContext().getVertexName() + " to " + oneToOneSrcTaskCount + " to match source 1-1 vertices."); getContext().reconfigureVertex(oneToOneSrcTaskCount, null, null); } oneToOneSrcTasksDoneCount = new int[oneToOneSrcTaskCount]; oneToOneLocationHints = new TaskLocationHint[oneToOneSrcTaskCount]; } - - Preconditions.checkState(numManagedTasks >=0, "Vertex: " + getContext().getVertexName()); + + Preconditions.checkState(numManagedTasks >= 0, "Vertex: " + getContext().getVertexName()); taskIsStarted = new boolean[numManagedTasks]; // allow scheduling @@ -135,11 +135,11 @@ private void configure() { getContext().doneReconfiguringVertex(); trySchedulingPendingCompletions(); } - + private boolean readyToSchedule() { return (configured.get() && started.get()); } - + private void trySchedulingPendingCompletions() { if (readyToSchedule() && !pendingCompletions.isEmpty()) { for (TaskAttemptIdentifier attempt : pendingCompletions) { @@ -147,7 +147,7 @@ private void trySchedulingPendingCompletions() { } } } - + @Override public void initialize() { // this will prevent vertex from starting until we notify we are done @@ -161,7 +161,7 @@ public void initialize() { getContext().registerForVertexStateUpdates(entry, EnumSet.of(VertexState.CONFIGURED)); } } - + @Override public synchronized void onVertexStateUpdated(VertexStateUpdate stateUpdate) throws Exception { numConfiguredSources++; @@ -183,7 +183,7 @@ public synchronized void onVertexStarted(List completions // allow scheduling started.set(true); - + trySchedulingPendingCompletions(); } @@ -205,9 +205,9 @@ public synchronized void onVertexManagerEventReceived(VertexManagerEvent vmEvent @Override public synchronized void onRootVertexInitialized(String inputName, - InputDescriptor inputDescriptor, List events) { + InputDescriptor inputDescriptor, List events) { } - + void handleSourceTaskFinished(String vertex, Integer taskId) { SourceVertexInfo srcInfo = srcVertexInfo.get(vertex); if (srcInfo.taskIsFinished[taskId.intValue()] == null) { @@ -221,7 +221,7 @@ void handleSourceTaskFinished(String vertex, Integer taskId) { oneToOneLocationHints[taskId.intValue()] = TaskLocationHint.createTaskLocationHint(vertex, taskId); } } - + // custom edge needs to tell us which of our tasks its connected to // for now only-built in edges supported // Check if current source task's vertex is completed. @@ -230,7 +230,7 @@ void handleSourceTaskFinished(String vertex, Integer taskId) { // we depend on all tasks to finish. So nothing to do now. return; } - + // currently finished vertex task may trigger us to schedule for (SourceVertexInfo vInfo : srcVertexInfo.values()) { if (vInfo.edgeProperty.getDataMovementType() != DataMovementType.ONE_TO_ONE) { @@ -241,7 +241,7 @@ void handleSourceTaskFinished(String vertex, Integer taskId) { } } } - + // all source vertices will full dependencies are done List tasksToStart = null; if (numOneToOneEdges == 0) { @@ -249,14 +249,14 @@ void handleSourceTaskFinished(String vertex, Integer taskId) { int numTasks = taskIsStarted.length; LOG.info("Starting all " + numTasks + "tasks for vertex: " + getContext().getVertexName()); tasksToStart = Lists.newArrayListWithCapacity(numTasks); - for (int i=0; i + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.library.vertexmanager; @@ -89,7 +89,6 @@ public class ShuffleVertexManager extends ShuffleVertexManagerBase { "tez.shuffle-vertex-manager.min-task-parallelism"; public static final int TEZ_SHUFFLE_VERTEX_MANAGER_MIN_TASK_PARALLELISM_DEFAULT = 1; - /** * In case of a ScatterGather connection, the fraction of source tasks which * should complete before tasks for the current vertex are scheduled @@ -115,21 +114,22 @@ public class ShuffleVertexManager extends ShuffleVertexManagerBase { private int basePartitionRange; private int remainderRangeForLastShuffler; - public ShuffleVertexManager(VertexManagerPluginContext context) { super(context); } static class ShuffleVertexManagerConfig extends ShuffleVertexManagerBaseConfig { final int minTaskParallelism; + public ShuffleVertexManagerConfig(final boolean enableAutoParallelism, - final long desiredTaskInputDataSize, final float slowStartMinFraction, - final float slowStartMaxFraction, final int minTaskParallelism) { + final long desiredTaskInputDataSize, final float slowStartMinFraction, + final float slowStartMaxFraction, final int minTaskParallelism) { super(enableAutoParallelism, desiredTaskInputDataSize, slowStartMinFraction, slowStartMaxFraction); this.minTaskParallelism = minTaskParallelism; LOG.info("minTaskParallelism {}", this.minTaskParallelism); } + int getMinTaskParallelism() { return minTaskParallelism; } @@ -152,15 +152,15 @@ ShuffleVertexManagerBaseConfig initConfiguration() { conf.getFloat( TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION, Math.max(slowStartMinFraction, - TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION_DEFAULT)), + TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION_DEFAULT)), Math.max(1, conf .getInt(TEZ_SHUFFLE_VERTEX_MANAGER_MIN_TASK_PARALLELISM, - TEZ_SHUFFLE_VERTEX_MANAGER_MIN_TASK_PARALLELISM_DEFAULT))); + TEZ_SHUFFLE_VERTEX_MANAGER_MIN_TASK_PARALLELISM_DEFAULT))); return mgrConfig; } static int[] createIndices(int partitionRange, int taskIndex, - int offSetPerTask) { + int offSetPerTask) { int startIndex = taskIndex * offSetPerTask; int[] indices = new int[partitionRange]; for (int currentIndex = 0; currentIndex < partitionRange; ++currentIndex) { @@ -175,7 +175,7 @@ public static class CustomShuffleEdgeManager extends EdgeManagerPluginOnDemand { int basePartitionRange; int remainderRangeForLastShuffler; int numSourceTasks; - + int[][] sourceIndices; int[][] targetIndices; @@ -210,7 +210,7 @@ public void initialize() { @Override public int getNumDestinationTaskPhysicalInputs(int destinationTaskIndex) { int partitionRange = 1; - if(destinationTaskIndex < numDestinationTasks-1) { + if (destinationTaskIndex < numDestinationTasks - 1) { partitionRange = basePartitionRange; } else { partitionRange = remainderRangeForLastShuffler; @@ -222,24 +222,24 @@ public int getNumDestinationTaskPhysicalInputs(int destinationTaskIndex) { public int getNumSourceTaskPhysicalOutputs(int sourceTaskIndex) { return numSourceTaskOutputs; } - + @Override public void routeDataMovementEventToDestination(DataMovementEvent event, - int sourceTaskIndex, int sourceOutputIndex, - Map> destinationTaskAndInputIndices) { + int sourceTaskIndex, int sourceOutputIndex, + Map> destinationTaskAndInputIndices) { int sourceIndex = event.getSourceIndex(); - int destinationTaskIndex = sourceIndex/basePartitionRange; + int destinationTaskIndex = sourceIndex / basePartitionRange; int partitionRange = 1; - if(destinationTaskIndex < numDestinationTasks-1) { + if (destinationTaskIndex < numDestinationTasks - 1) { partitionRange = basePartitionRange; } else { partitionRange = remainderRangeForLastShuffler; } // all inputs from a source task are next to each other in original order - int targetIndex = - sourceTaskIndex * partitionRange - + sourceIndex % partitionRange; + int targetIndex = + sourceTaskIndex * partitionRange + + sourceIndex % partitionRange; destinationTaskAndInputIndices.put( destinationTaskIndex, Collections.singletonList(targetIndex)); @@ -249,41 +249,40 @@ public void routeDataMovementEventToDestination(DataMovementEvent event, public EventRouteMetadata routeDataMovementEventToDestination( int sourceTaskIndex, int sourceOutputIndex, int destTaskIndex) throws Exception { int sourceIndex = sourceOutputIndex; - int destinationTaskIndex = sourceIndex/basePartitionRange; + int destinationTaskIndex = sourceIndex / basePartitionRange; if (destinationTaskIndex != destTaskIndex) { return null; } int partitionRange = 1; - if(destinationTaskIndex < numDestinationTasks-1) { + if (destinationTaskIndex < numDestinationTasks - 1) { partitionRange = basePartitionRange; } else { partitionRange = remainderRangeForLastShuffler; } - + // all inputs from a source task are next to each other in original order - int targetIndex = - sourceTaskIndex * partitionRange - + sourceIndex % partitionRange; + int targetIndex = + sourceTaskIndex * partitionRange + + sourceIndex % partitionRange; return EventRouteMetadata.create(1, new int[]{targetIndex}); } - @Override public void prepareForRouting() throws Exception { // target indices derive from num src tasks int numSourceTasks = getContext().getSourceVertexNumTasks(); targetIndices = new int[numSourceTasks][]; - for (int srcTaskIndex=0; srcTaskIndex> destinationTaskAndInputIndices) { + public void routeInputSourceTaskFailedEventToDestination(int sourceTaskIndex, + Map> destinationTaskAndInputIndices) { if (remainderRangeForLastShuffler < basePartitionRange) { int startOffset = sourceTaskIndex * basePartitionRange; List allIndices = Lists.newArrayListWithCapacity(basePartitionRange); - for (int i=0; i inputIndices = Collections.unmodifiableList(allIndices); - for (int i=0; i allIndices = Lists.newArrayListWithCapacity(basePartitionRange); - for (int i=0; i inputIndices = Collections.unmodifiableList(allIndices); - for (int i=0; i 0) { + if (bigDesiredTaskParallelism.compareTo(BigInteger.valueOf(Integer.MAX_VALUE)) > 0) { LOG.info("Not reducing auto parallelism for vertex: {}" + " since the desired parallelism of {} is greater than or equal" + " to the max parallelism of {}", getContext().getVertexName(), @@ -471,25 +468,25 @@ ReconfigVertexParams computeRouting() { return null; } int desiredTaskParallelism = bigDesiredTaskParallelism.intValue(); - if(desiredTaskParallelism < mgrConfig.getMinTaskParallelism()) { + if (desiredTaskParallelism < mgrConfig.getMinTaskParallelism()) { desiredTaskParallelism = mgrConfig.getMinTaskParallelism(); } - if(desiredTaskParallelism >= currentParallelism) { + if (desiredTaskParallelism >= currentParallelism) { LOG.info("Not reducing auto parallelism for vertex: {}" - + " since the desired parallelism of {} is greater than or equal" - + " to the current parallelism of {}", getContext().getVertexName(), + + " since the desired parallelism of {} is greater than or equal" + + " to the current parallelism of {}", getContext().getVertexName(), desiredTaskParallelism, pendingTasks.size()); return null; } // most shufflers will be assigned this range - basePartitionRange = currentParallelism/desiredTaskParallelism; + basePartitionRange = currentParallelism / desiredTaskParallelism; if (basePartitionRange <= 1) { // nothing to do if range is equal 1 partition. shuffler does it by default LOG.info("Not reducing auto parallelism for vertex: {} by less than" - + " half since combining two inputs will potentially break the" - + " desired task input size of {}", getContext().getVertexName(), + + " half since combining two inputs will potentially break the" + + " desired task input size of {}", getContext().getVertexName(), config.getDesiredTaskInputDataSize()); return null; } @@ -503,7 +500,7 @@ ReconfigVertexParams computeRouting() { getContext().getVertexName(), pendingTasks.size(), finalTaskParallelism); - if(finalTaskParallelism >= currentParallelism) { + if (finalTaskParallelism >= currentParallelism) { return null; } @@ -511,13 +508,13 @@ ReconfigVertexParams computeRouting() { new CustomShuffleEdgeManagerConfig( currentParallelism, finalTaskParallelism, basePartitionRange, ((remainderRangeForLastShuffler > 0) ? - remainderRangeForLastShuffler : basePartitionRange)); + remainderRangeForLastShuffler : basePartitionRange)); EdgeManagerPluginDescriptor descriptor = EdgeManagerPluginDescriptor.create(CustomShuffleEdgeManager.class.getName()); descriptor.setUserPayload(edgeManagerConfig.toUserPayload()); Iterable> bipartiteItr = getBipartiteInfo(); - for(Map.Entry entry : bipartiteItr) { + for (Map.Entry entry : bipartiteItr) { entry.getValue().newDescriptor = descriptor; } ReconfigVertexParams params = @@ -527,7 +524,7 @@ ReconfigVertexParams computeRouting() { @Override void postReconfigVertex() { - configureTargetMapping(pendingTasks.size()); + configureTargetMapping(pendingTasks.size()); } private void configureTargetMapping(int tasks) { @@ -615,7 +612,7 @@ private boolean computePartitionSizes() { int index = taskInfo.getIndex(); if (targetIndexes != null) { //parallelism has changed. Preconditions.checkState(index < targetIndexes.length, - "index=" + index +", targetIndexes length=" + targetIndexes.length); + "index=" + index + ", targetIndexes length=" + targetIndexes.length); int[] mapping = targetIndexes[index]; int partitionStats = 0; for (int i : mapping) { @@ -630,12 +627,10 @@ private boolean computePartitionSizes() { return computedPartitionSizes; } - - /** * Create a {@link VertexManagerPluginDescriptor} builder that can be used to * configure the plugin. - * + * * @param conf * {@link Configuration} May be modified in place. May be null if the * configuration parameters are to be set only via code. If @@ -672,13 +667,13 @@ public ShuffleVertexManagerConfigBuilder setAutoReduceParallelism( } public ShuffleVertexManagerConfigBuilder - setSlowStartMinSrcCompletionFraction(float minFraction) { + setSlowStartMinSrcCompletionFraction(float minFraction) { conf.setFloat(TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, minFraction); return this; } public ShuffleVertexManagerConfigBuilder - setSlowStartMaxSrcCompletionFraction(float maxFraction) { + setSlowStartMaxSrcCompletionFraction(float maxFraction) { conf.setFloat(TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION, maxFraction); return this; } @@ -693,7 +688,7 @@ public ShuffleVertexManagerConfigBuilder setDesiredTaskInputSize( public ShuffleVertexManagerConfigBuilder setMinTaskParallelism( int minTaskParallelism) { conf.setInt(TEZ_SHUFFLE_VERTEX_MANAGER_MIN_TASK_PARALLELISM, - minTaskParallelism); + minTaskParallelism); return this; } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManagerBase.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManagerBase.java index 05e2d8ce78..a587f9cf20 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManagerBase.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManagerBase.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.dag.library.vertexmanager; @@ -55,7 +55,6 @@ import org.apache.tez.runtime.api.events.VertexManagerEvent; import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.VertexManagerEventPayloadProto; - import java.io.DataInputStream; import java.io.IOException; import java.math.BigInteger; @@ -79,7 +78,7 @@ abstract class ShuffleVertexManagerBase extends VertexManagerPlugin { static long MB = 1024l * 1024l; private static final Logger LOG = - LoggerFactory.getLogger(ShuffleVertexManagerBase.class); + LoggerFactory.getLogger(ShuffleVertexManagerBase.class); ComputeRoutingAction computeRoutingAction = ComputeRoutingAction.WAIT; @@ -135,7 +134,7 @@ static class SourceVertexInfo { EdgeManagerPluginDescriptor newDescriptor; SourceVertexInfo(final EdgeProperty edgeProperty, - int totalTasksToSchedule) { + int totalTasksToSchedule) { this.edgeProperty = edgeProperty; this.finishedTaskSet = new BitSet(); this.statsInMB = new int[totalTasksToSchedule]; @@ -151,16 +150,16 @@ int getNumCompletedTasks() { BigInteger getExpectedStatsAtIndex(int index) { return (numVMEventsReceived == 0) ? - BigInteger.ZERO : - BigInteger.valueOf(statsInMB[index]). - multiply(BigInteger.valueOf(numTasks)). - divide(BigInteger.valueOf(numVMEventsReceived)). - multiply(BigInteger.valueOf(MB)); + BigInteger.ZERO : + BigInteger.valueOf(statsInMB[index]). + multiply(BigInteger.valueOf(numTasks)). + divide(BigInteger.valueOf(numVMEventsReceived)). + multiply(BigInteger.valueOf(MB)); } } SourceVertexInfo createSourceVertexInfo(EdgeProperty edgeProperty, - int numTasks) { + int numTasks) { return new SourceVertexInfo(edgeProperty, numTasks); } @@ -175,12 +174,15 @@ public PendingTaskInfo(int index) { public String toString() { return "[index=" + index + ", inputStats=" + inputStats + "]"; } + public int getIndex() { return index; } + public int getInputStats() { return inputStats; } + // return true if stat is set. public boolean setInputStats(int inputStats) { if (inputStats > 0 && this.inputStats != inputStats) { @@ -197,7 +199,7 @@ static class ReconfigVertexParams { final private VertexLocationHint locationHint; public ReconfigVertexParams(final int finalParallelism, - final VertexLocationHint locationHint) { + final VertexLocationHint locationHint) { this.finalParallelism = finalParallelism; this.locationHint = locationHint; } @@ -205,6 +207,7 @@ public ReconfigVertexParams(final int finalParallelism, public int getFinalParallelism() { return finalParallelism; } + public VertexLocationHint getLocationHint() { return locationHint; } @@ -219,7 +222,7 @@ public ShuffleVertexManagerBase(VertexManagerPluginContext context) { public synchronized void onVertexStarted(List completions) { // examine edges after vertex started because until then these may not have been defined Map inputs = getContext().getInputVertexEdgeProperties(); - for(Map.Entry entry : inputs.entrySet()) { + for (Map.Entry entry : inputs.entrySet()) { srcVertexInfo.put(entry.getKey(), createSourceVertexInfo(entry.getValue(), getContext().getVertexNumTasks(getContext().getVertexName()))); // TODO what if derived class has already called this @@ -246,7 +249,7 @@ public synchronized void onVertexStarted(List completions pendingVMEvents.clear(); LOG.info("OnVertexStarted vertex: {} with {} source tasks and {} pending" + - " tasks", getContext().getVertexName(), totalNumBipartiteSourceTasks, + " tasks", getContext().getVertexName(), totalNumBipartiteSourceTasks, totalTasksToSchedule); if (completions != null) { @@ -260,7 +263,7 @@ public synchronized void onVertexStarted(List completions } protected void onVertexStartedCheck() { - if(bipartiteSources == 0) { + if (bipartiteSources == 0) { throw new TezUncheckedException("At least 1 bipartite source should exist"); } } @@ -273,7 +276,7 @@ public synchronized void onSourceTaskCompleted(TaskAttemptIdentifier attempt) { if (srcInfo.vertexIsConfigured) { Preconditions.checkState(srcTaskId < srcInfo.numTasks, "Received completion for srcTaskId " + srcTaskId + " but Vertex: " + srcVertexName + - " has only " + srcInfo.numTasks + " tasks"); + " has only " + srcInfo.numTasks + " tasks"); } //handle duplicate events and count task completions from all source vertices BitSet completedSourceTasks = srcInfo.finishedTaskSet; @@ -290,7 +293,7 @@ public synchronized void onSourceTaskCompleted(TaskAttemptIdentifier attempt) { @VisibleForTesting void parsePartitionStats(SourceVertexInfo srcInfo, - RoaringBitmap partitionStats) { + RoaringBitmap partitionStats) { Preconditions.checkState(srcInfo.statsInMB != null, "Stats should be initialized"); Iterator it = partitionStats.iterator(); @@ -308,8 +311,8 @@ void parsePartitionStats(SourceVertexInfo srcInfo, } void parseDetailedPartitionStats(SourceVertexInfo srcInfo, - List partitionStats) { - for (int i=0; i partitionStats) { + for (int i = 0; i < partitionStats.size(); i++) { srcInfo.statsInMB[i] += partitionStats.get(i); } } @@ -363,7 +366,6 @@ private void handleVertexManagerEvent(VertexManagerEvent vmEvent) { partitionStats.deserialize(new DataInputStream(bin)); parsePartitionStats(srcInfo, partitionStats); - } catch (IOException e) { throw new TezUncheckedException(e); } @@ -379,11 +381,11 @@ private void handleVertexManagerEvent(VertexManagerEvent vmEvent) { if (LOG.isDebugEnabled()) { LOG.debug("For attempt: {} received info of output size: {}" - + " vertex numEventsReceived: {} vertex output size: {}" - + " total numEventsReceived: {} total output size: {}", - vmEvent.getProducerAttemptIdentifier(), sourceTaskOutputSize, - srcInfo.numVMEventsReceived, srcInfo.outputSize, - numVertexManagerEventsReceived, completedSourceTasksOutputSize); + + " vertex numEventsReceived: {} vertex output size: {}" + + " total numEventsReceived: {} total output size: {}", + vmEvent.getProducerAttemptIdentifier(), sourceTaskOutputSize, + srcInfo.numVMEventsReceived, srcInfo.outputSize, + numVertexManagerEventsReceived, completedSourceTasksOutputSize); } } @@ -412,13 +414,13 @@ private ComputeRoutingAction getComputeRoutingAction( // Wait when there aren't enough completed tasks return ComputeRoutingAction.WAIT; } else if (numVertexManagerEventsReceived == 0 && - totalNumBipartiteSourceTasks > 0) { + totalNumBipartiteSourceTasks > 0) { // When source tasks don't have output data, // there will be no VME. return ComputeRoutingAction.SKIP; } else if ( completedSourceTasksOutputSize < config.getDesiredTaskInputDataSize() - && (minSourceVertexCompletedTaskFraction < config.getMaxFraction())) { + && (minSourceVertexCompletedTaskFraction < config.getMaxFraction())) { /** * When overall completed output size is not even equal to * desiredTaskInputSize, we can wait for some more data to be available to @@ -427,16 +429,16 @@ private ComputeRoutingAction getComputeRoutingAction( * honored strictly in this case. */ LOG.info("Defer scheduling tasks; vertex = {}" - + ", totalNumBipartiteSourceTasks = {}" - + ", completedSourceTasksOutputSize = {}" - + ", numVertexManagerEventsReceived = {}" - + ", numBipartiteSourceTasksCompleted = {}" - + ", minSourceVertexCompletedTaskFraction = {}", + + ", totalNumBipartiteSourceTasks = {}" + + ", completedSourceTasksOutputSize = {}" + + ", numVertexManagerEventsReceived = {}" + + ", numBipartiteSourceTasksCompleted = {}" + + ", minSourceVertexCompletedTaskFraction = {}", getContext().getVertexName(), totalNumBipartiteSourceTasks, completedSourceTasksOutputSize, numVertexManagerEventsReceived, numBipartiteSourceTasksCompleted, minSourceVertexCompletedTaskFraction); - return ComputeRoutingAction.WAIT; + return ComputeRoutingAction.WAIT; } else { return ComputeRoutingAction.COMPUTE; } @@ -463,7 +465,7 @@ BigInteger getExpectedTotalBipartiteSourceTasksOutputSize() { int getCurrentlyKnownStatsAtIndex(int index) { int stats = 0; - for(SourceVertexInfo entry : getAllSourceVertexInfo()) { + for (SourceVertexInfo entry : getAllSourceVertexInfo()) { stats += entry.statsInMB[index]; } return stats; @@ -471,7 +473,7 @@ int getCurrentlyKnownStatsAtIndex(int index) { long getExpectedStatsAtIndex(int index) { BigInteger stats = BigInteger.ZERO; - for(SourceVertexInfo entry : getAllSourceVertexInfo()) { + for (SourceVertexInfo entry : getAllSourceVertexInfo()) { stats = stats.add(entry.getExpectedStatsAtIndex(index)); } if (stats.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) { @@ -523,7 +525,6 @@ private boolean determineParallelismAndApply() { * End of functions related to how new parallelism is determined. */ - /** * Subclass might return null or empty list to indicate no tasks * to schedule at this point. @@ -554,12 +555,12 @@ SourceVertexInfo getSourceVertexInfo(String vertextName) { Iterable> getBipartiteInfo() { return Iterables.filter(srcVertexInfo.entrySet(), - new Predicate>() { - public boolean apply(Map.Entry input) { - return (input.getValue().edgeProperty.getDataMovementType() == - DataMovementType.SCATTER_GATHER); - } - }); + new Predicate>() { + public boolean apply(Map.Entry input) { + return (input.getValue().edgeProperty.getDataMovementType() == + DataMovementType.SCATTER_GATHER); + } + }); } /** @@ -568,7 +569,7 @@ public boolean apply(Map.Entry input) { * @return boolean */ private boolean canScheduleTasks() { - for(Map.Entry entry : srcVertexInfo.entrySet()) { + for (Map.Entry entry : srcVertexInfo.entrySet()) { // need to check for vertex configured because until that we dont know // if numTask s== 0 is valid if (!entry.getValue().vertexIsConfigured) { // isConfigured @@ -590,8 +591,8 @@ int getNumOfTasksToScheduleAndLog(float minFraction) { // numTasksToSchedule can be -ve if minFraction // is less than slowStartMinSrcCompletionFraction. LOG.info("Scheduling {} tasks for vertex: {} with totalTasks: {}. " + - "{} source tasks completed out of {}. " + - "MinSourceTaskCompletedFraction: {} min: {} max: {}", + "{} source tasks completed out of {}. " + + "MinSourceTaskCompletedFraction: {} min: {} max: {}", numTasksToSchedule, getContext().getVertexName(), totalTasksToSchedule, numBipartiteSourceTasksCompleted, totalNumBipartiteSourceTasks, minFraction, @@ -620,7 +621,7 @@ int getNumOfTasksToSchedule(float minSourceVertexCompletedTaskFraction) { config.getMinFraction()) / percentRange; } else { // min and max are equal. schedule 100% on reaching min - if(minSourceVertexCompletedTaskFraction < + if (minSourceVertexCompletedTaskFraction < config.getMinFraction()) { tasksFractionToSchedule = 0; } @@ -631,7 +632,7 @@ int getNumOfTasksToSchedule(float minSourceVertexCompletedTaskFraction) { // round up to avoid the corner case that single task cannot be scheduled // until src completed fraction reach max - return ((int)(Math.ceil(tasksFractionToSchedule * totalTasksToSchedule)) - + return ((int) (Math.ceil(tasksFractionToSchedule * totalTasksToSchedule)) - (totalTasksToSchedule - numPendingTasks)); } @@ -657,7 +658,6 @@ float getMinSourceVertexCompletedTaskFraction() { return minSourceVertexCompletedTaskFraction; } - private boolean preconditionsSatisfied() { if (!onVertexStartedDone.get()) { // vertex not started yet @@ -695,7 +695,7 @@ private void processPendingTasks(TaskAttemptIdentifier completedSourceAttempt) { // as specified by the user. If/When we move to some other method of // calculating parallelism or change parallelism while tasks are already // running then we can create other parameters to trigger this calculation. - if(config.isAutoParallelismEnabled()) { + if (config.isAutoParallelismEnabled()) { if (!determineParallelismAndApply()) { //try to determine parallelism later when more info is available. return; @@ -710,9 +710,10 @@ static class ShuffleVertexManagerBaseConfig { final private long desiredTaskInputDataSize; final private float slowStartMinFraction; final private float slowStartMaxFraction; + public ShuffleVertexManagerBaseConfig(final boolean enableAutoParallelism, - final long desiredTaskInputDataSize, final float slowStartMinFraction, - final float slowStartMaxFraction) { + final long desiredTaskInputDataSize, final float slowStartMinFraction, + final float slowStartMaxFraction) { if (slowStartMinFraction < 0 || slowStartMaxFraction > 1 || slowStartMaxFraction < slowStartMinFraction) { throw new IllegalArgumentException( @@ -736,12 +737,15 @@ public ShuffleVertexManagerBaseConfig(final boolean enableAutoParallelism, public boolean isAutoParallelismEnabled() { return this.enableAutoParallelism; } + public long getDesiredTaskInputDataSize() { return this.desiredTaskInputDataSize; } + public float getMinFraction() { return this.slowStartMinFraction; } + public float getMaxFraction() { return this.slowStartMaxFraction; } @@ -780,7 +784,7 @@ private void handleVertexStateUpdate(VertexStateUpdate stateUpdate) { totalNumBipartiteSourceTasks += vInfo.numTasks; } LOG.info("Received configured notification : {}" + " for vertex: {} in" + - " vertex: {}" + " numBipartiteSourceTasks: {}", + " vertex: {}" + " numBipartiteSourceTasks: {}", stateUpdate.getVertexState(), stateUpdate.getVertexName(), getContext().getVertexName(), totalNumBipartiteSourceTasks); processPendingTasks(null); @@ -812,7 +816,7 @@ public synchronized void onVertexStateUpdated(VertexStateUpdate stateUpdate) { @Override public synchronized void onRootVertexInitialized(String inputName, - InputDescriptor inputDescriptor, List events) { + InputDescriptor inputDescriptor, List events) { // Not allowing this for now. Nothing to do. } @@ -820,7 +824,7 @@ private void reconfigVertex(final int finalTaskParallelism) { Map edgeProperties = new HashMap(bipartiteSources); Iterable> bipartiteItr = getBipartiteInfo(); - for(Map.Entry entry : bipartiteItr) { + for (Map.Entry entry : bipartiteItr) { String vertex = entry.getKey(); EdgeProperty oldEdgeProp = entry.getValue().edgeProperty; EdgeProperty newEdgeProp = EdgeProperty.create(entry.getValue().newDescriptor, diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/VertexManagerWithConcurrentInput.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/VertexManagerWithConcurrentInput.java index caf5acd588..2644e58e2d 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/VertexManagerWithConcurrentInput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/VertexManagerWithConcurrentInput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -146,7 +146,7 @@ public synchronized void onVertexStateUpdated(VertexStateUpdate stateUpdate) { @Override public synchronized void onSourceTaskCompleted(TaskAttemptIdentifier attempt) { - completedUpstreamTasks ++; + completedUpstreamTasks++; LOG.info("Source task attempt {} completion received at vertex {}", attempt, this.vertexName); } @@ -195,7 +195,6 @@ private boolean canScheduleTasks() { } } - /** * Create a {@link VertexManagerPluginDescriptor} builder that can be used to * configure the plugin. @@ -241,5 +240,4 @@ public VertexManagerPluginDescriptor build() { } } } - } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/http/BaseHttpConnection.java b/tez-runtime-library/src/main/java/org/apache/tez/http/BaseHttpConnection.java index dd642ae4f5..e53994311c 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/http/BaseHttpConnection.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/http/BaseHttpConnection.java @@ -59,5 +59,4 @@ public abstract class BaseHttpConnection { * @throws IOException */ public abstract void cleanup(boolean disconnect) throws IOException; - } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/http/HttpConnection.java b/tez-runtime-library/src/main/java/org/apache/tez/http/HttpConnection.java index e12331c250..35487fd77c 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/http/HttpConnection.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/http/HttpConnection.java @@ -70,7 +70,7 @@ public class HttpConnection extends BaseHttpConnection { * @throws IOException */ public HttpConnection(URL url, HttpConnectionParams connParams, - String logIdentifier, JobTokenSecretManager jobTokenSecretManager) throws IOException { + String logIdentifier, JobTokenSecretManager jobTokenSecretManager) throws IOException { this.logIdentifier = logIdentifier; this.jobTokenSecretMgr = jobTokenSecretManager; this.httpConnParams = connParams; @@ -188,7 +188,6 @@ private boolean connect(int connectionTimeout) throws IOException { // reset the connect time out for the final connect connection.setConnectTimeout(unit); } - } } if (LOG.isDebugEnabled()) { @@ -255,7 +254,7 @@ public DataInputStream getInputStream() throws IOException { stopWatch.reset().start(); if (connectionSucceeed) { input = new DataInputStream(new BufferedInputStream( - connection.getInputStream(), httpConnParams.getBufferSize())); + connection.getInputStream(), httpConnParams.getBufferSize())); } if (LOG.isDebugEnabled()) { LOG.debug("Time taken to getInputStream (connect) " + url + @@ -325,4 +324,3 @@ private void readErrorStream(InputStream errorStream) { } } } - diff --git a/tez-runtime-library/src/main/java/org/apache/tez/http/HttpConnectionParams.java b/tez-runtime-library/src/main/java/org/apache/tez/http/HttpConnectionParams.java index 9f1c002a20..a1741dc2a6 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/http/HttpConnectionParams.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/http/HttpConnectionParams.java @@ -30,7 +30,7 @@ public class HttpConnectionParams { public HttpConnectionParams(boolean keepAlive, int keepAliveMaxConnections, int connectionTimeout, int readTimeout, int bufferSize, boolean sslShuffle, SSLFactory - sslFactory) { + sslFactory) { this.keepAlive = keepAlive; this.keepAliveMaxConnections = keepAliveMaxConnections; this.connectionTimeout = connectionTimeout; @@ -68,7 +68,6 @@ public SSLFactory getSslFactory() { return sslFactory; } - public String toString() { StringBuilder sb = new StringBuilder(); sb.append("keepAlive=").append(keepAlive).append(", "); diff --git a/tez-runtime-library/src/main/java/org/apache/tez/http/SSLFactory.java b/tez-runtime-library/src/main/java/org/apache/tez/http/SSLFactory.java index 8c98764bfe..bd4545f0b3 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/http/SSLFactory.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/http/SSLFactory.java @@ -171,7 +171,6 @@ public KeyStoresFactory getKeystoresFactory() { return keystoresFactory; } - /** * Returns a configured SSLSocketFactory. * @@ -200,8 +199,6 @@ public HostnameVerifier getHostnameVerifier() { return hostnameVerifier; } - - /** * If the given {@link HttpURLConnection} is an {@link HttpsURLConnection} * configures the connection with the {@link SSLSocketFactory} and @@ -243,4 +240,4 @@ public void configure(DefaultAsyncHttpClientConfig.Builder builder) throws IOExc builder.setSslContext(jdkSslContext); } } -} \ No newline at end of file +} diff --git a/tez-runtime-library/src/main/java/org/apache/tez/http/async/netty/AsyncHttpConnection.java b/tez-runtime-library/src/main/java/org/apache/tez/http/async/netty/AsyncHttpConnection.java index 215e63af58..4c735584e4 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/http/async/netty/AsyncHttpConnection.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/http/async/netty/AsyncHttpConnection.java @@ -119,7 +119,7 @@ private void initClient(HttpConnectionParams httpConnParams) throws IOException } public AsyncHttpConnection(URL url, HttpConnectionParams connParams, - String logIdentifier, JobTokenSecretManager jobTokenSecretManager) throws IOException { + String logIdentifier, JobTokenSecretManager jobTokenSecretManager) throws IOException { this.jobTokenSecretMgr = jobTokenSecretManager; this.httpConnParams = connParams; this.url = url; @@ -169,7 +169,7 @@ public boolean connect() throws IOException, InterruptedException { if (response == null) { throw new IOException("Response is null"); } - } catch(IOException e) { + } catch (IOException e) { throw e; } @@ -222,6 +222,7 @@ public void close() throws IOException { httpAsyncClient.close(); httpAsyncClient = null; } + /** * Cleanup the connection. * @@ -237,5 +238,4 @@ public void cleanup(boolean disconnect) throws IOException { IOUtils.closeQuietly(pis); response = null; } - } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/http/async/netty/TezBodyDeferringAsyncHandler.java b/tez-runtime-library/src/main/java/org/apache/tez/http/async/netty/TezBodyDeferringAsyncHandler.java index acbdab7df9..ed232390b9 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/http/async/netty/TezBodyDeferringAsyncHandler.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/http/async/netty/TezBodyDeferringAsyncHandler.java @@ -207,7 +207,7 @@ static class BodyDeferringInputStream extends FilterInputStream { private final TezBodyDeferringAsyncHandler bdah; public BodyDeferringInputStream(final Future future, - final TezBodyDeferringAsyncHandler bdah, final InputStream in) { + final TezBodyDeferringAsyncHandler bdah, final InputStream in) { super(in); this.future = future; this.bdah = bdah; @@ -255,4 +255,4 @@ public Response getLastResponse() throws InterruptedException, ExecutionExceptio return future.get(); } } -} \ No newline at end of file +} diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValueReader.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValueReader.java index 47f335bc72..0d2c28f055 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValueReader.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValueReader.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,7 +26,7 @@ /** * A key/value(s) pair based {@link Reader}. - * + * * Example usage * * while (kvReader.next()) { @@ -45,7 +45,7 @@ public abstract class KeyValueReader extends Reader { /** * Moves to the next key/values(s) pair - * + * * @return true if another key/value(s) pair exists, false if there are no more. * @throws IOException * if an error occurs @@ -53,14 +53,12 @@ public abstract class KeyValueReader extends Reader { */ public abstract boolean next() throws IOException; - /** * Returns the current key * @return the current key */ public abstract Object getCurrentKey() throws IOException; - - + /** * Returns the current value * @return the current value diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValueWriter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValueWriter.java index b5c4294a13..eb52c29dad 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValueWriter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValueWriter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -32,7 +32,7 @@ public abstract class KeyValueWriter extends Writer { /** * Writes a key/value pair. - * + * * @param key * the key to write * @param value diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValueWriterWithBasePath.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValueWriterWithBasePath.java index 5446ca6f8b..f8e64e0785 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValueWriterWithBasePath.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValueWriterWithBasePath.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValuesReader.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValuesReader.java index 7760818876..16233697ed 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValuesReader.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValuesReader.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,7 +26,7 @@ /** * A key/value(s) pair based {@link Reader}. - * + * * Example usage * * while (kvReader.next()) { @@ -45,7 +45,7 @@ public abstract class KeyValuesReader extends Reader { /** * Moves to the next key/values(s) pair - * + * * @return true if another key/value(s) pair exists, false if there are no more. * @throws IOException * if an error occurs @@ -53,13 +53,12 @@ public abstract class KeyValuesReader extends Reader { */ public abstract boolean next() throws IOException; - /** * Returns the current key * @return the current key */ public abstract Object getCurrentKey() throws IOException; - + /** * Returns an Iterable view of the values associated with the current key * @return an Iterable view of the values associated with the current key diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValuesWriter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValuesWriter.java index 9cdde435db..a27208a331 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValuesWriter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/KeyValuesWriter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/Partitioner.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/Partitioner.java index aa73e3bd4c..ea5ed50190 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/Partitioner.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/Partitioner.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,25 +24,25 @@ /** * {@link Partitioner} is used by the TEZ framework to partition output * key/value pairs. - * + * * Partitioner Initialization

    The Partitioner class is picked up * using the TEZ_RUNTIME_PARTITIONER_CLASS attribute in {@link TezRuntimeConfiguration} - * + * * TODO NEWTEZ Change construction to first check for a Constructor with a bytep[] payload - * + * * Partitioners need to provide a single argument ({@link Configuration}) * constructor or a 0 argument constructor. If both exist, preference is given * to the single argument constructor. This is primarily for MR support. - * + * * If using the configuration constructor, TEZ_RUNTIME_NUM_EXPECTED_PARTITIONS * will be set in the configuration, to indicate the max number of expected * partitions. - * + * */ @Public @Evolving public interface Partitioner { - + /** * Get partition for given key/value. * @param key key @@ -51,5 +51,4 @@ public interface Partitioner { * @return partition for the given key/value */ int getPartition(Object key, Object value, int numPartitions); - } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java index 1c747af2bb..ec226027da 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/api/TezRuntimeConfiguration.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -65,7 +65,6 @@ public class TezRuntimeConfiguration { private static final List allowedPrefixes = new ArrayList(); private static List unmodifiableAllowedPrefixes; - static { Configuration.addDeprecation("tez.runtime.sort.threads", TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SORTER_SORT_THREADS); @@ -97,31 +96,26 @@ public class TezRuntimeConfiguration { */ public static final String TEZ_RUNTIME_IO_FILE_BUFFER_SIZE = "io.file.buffer.size"; - @ConfigurationProperty(type = "integer") public static final String TEZ_RUNTIME_IO_SORT_FACTOR = TEZ_RUNTIME_PREFIX + "io.sort.factor"; public static final int TEZ_RUNTIME_IO_SORT_FACTOR_DEFAULT = 100; - @ConfigurationProperty(type = "float") public static final String TEZ_RUNTIME_SORT_SPILL_PERCENT = TEZ_RUNTIME_PREFIX + "sort.spill.percent"; public static final float TEZ_RUNTIME_SORT_SPILL_PERCENT_DEFAULT = 0.8f; - @ConfigurationProperty(type = "integer") public static final String TEZ_RUNTIME_IO_SORT_MB = TEZ_RUNTIME_PREFIX + "io.sort.mb"; public static final int TEZ_RUNTIME_IO_SORT_MB_DEFAULT = 100; - @ConfigurationProperty(type = "integer") public static final String TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES = TEZ_RUNTIME_PREFIX + "index.cache.memory.limit.bytes"; public static final int TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES_DEFAULT = 1024 * 1024; - // TODO Use the default value @ConfigurationProperty(type = "integer") public static final String TEZ_RUNTIME_COMBINE_MIN_SPILLS = TEZ_RUNTIME_PREFIX + @@ -176,7 +170,7 @@ public class TezRuntimeConfiguration { * Integer value. Percentage of buffer to be filled before we spill to disk. Default value is 0, * which will spill for every buffer. */ - @ConfigurationProperty(type="int") + @ConfigurationProperty(type = "int") public static final String TEZ_RUNTIME_UNORDERED_PARTITIONED_KVWRITER_BUFFER_MERGE_PERCENT = TEZ_RUNTIME_PREFIX + "unordered-partitioned-kvwriter.buffer-merge-percent"; public static final int TEZ_RUNTIME_UNORDERED_PARTITIONED_KVWRITER_BUFFER_MERGE_PERCENT_DEFAULT = @@ -346,7 +340,6 @@ public class TezRuntimeConfiguration { public static final boolean TEZ_RUNTIME_SHUFFLE_FAILED_CHECK_SINCE_LAST_COMPLETION_DEFAULT = true; - @ConfigurationProperty(type = "integer") public static final String TEZ_RUNTIME_SHUFFLE_FETCH_MAX_TASK_OUTPUT_AT_ONCE = TEZ_RUNTIME_PREFIX + @@ -471,7 +464,6 @@ public class TezRuntimeConfiguration { "task.input.post-merge.buffer.percent"; public static final float TEZ_RUNTIME_INPUT_BUFFER_PERCENT_DEFAULT = 0.0f; - @ConfigurationProperty public static final String TEZ_RUNTIME_GROUP_COMPARATOR_CLASS = TEZ_RUNTIME_PREFIX + "group.comparator.class"; @@ -509,13 +501,13 @@ public class TezRuntimeConfiguration { @Private public static final String TEZ_RUNTIME_TRANSFER_DATA_VIA_EVENTS_ENABLED = - TEZ_RUNTIME_PREFIX + "transfer.data-via-events.enabled"; + TEZ_RUNTIME_PREFIX + "transfer.data-via-events.enabled"; @Private public static final boolean TEZ_RUNTIME_TRANSFER_DATA_VIA_EVENTS_ENABLED_DEFAULT = true; @Private public static final String TEZ_RUNTIME_TRANSFER_DATA_VIA_EVENTS_MAX_SIZE = - TEZ_RUNTIME_PREFIX + "transfer.data-via-events.max-size"; + TEZ_RUNTIME_PREFIX + "transfer.data-via-events.max-size"; @Private public static final int TEZ_RUNTIME_TRANSFER_DATA_VIA_EVENTS_MAX_SIZE_DEFAULT = 512; @@ -560,11 +552,10 @@ public class TezRuntimeConfiguration { * before sending the events in milliseconds. Set to -1 to not wait. */ @ConfigurationProperty(type = "integer") - public static final String TEZ_RUNTIME_SHUFFLE_BATCH_WAIT = + public static final String TEZ_RUNTIME_SHUFFLE_BATCH_WAIT = TEZ_RUNTIME_PREFIX + "shuffle.batch.wait"; public static final int TEZ_RUNTIME_SHUFFLE_BATCH_WAIT_DEFAULT = -1; - /** * Share data fetched between tasks running on the same host if applicable. */ @@ -737,7 +728,7 @@ public enum ReportPartitionStats { * Configuration @link{#TEZ_RUNTIME_REPORT_PARTITION_STATS} used * to be boolean type. */ - DISABLED("false"), + DISABLED("false"), @Deprecated /** @@ -746,7 +737,7 @@ public enum ReportPartitionStats { * Configuration @link{#TEZ_RUNTIME_REPORT_PARTITION_STATS} used * to be boolean type. */ - ENABLED("true"), + ENABLED("true"), /** * Don't report partition stats. @@ -776,7 +767,7 @@ public final String getType() { public boolean isEnabled() { return !equals(ReportPartitionStats.DISABLED) && - !equals(ReportPartitionStats.NONE); + !equals(ReportPartitionStats.NONE); } public boolean isPrecise() { diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductCombination.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductCombination.java index c1eae7019e..2a3fd5aef6 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductCombination.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductCombination.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -54,13 +54,13 @@ class CartesianProductCombination { public CartesianProductCombination(int[] numChunk) { Preconditions.checkArgument(!Ints.contains(numChunk, 0), - "CartesianProductCombination doesn't allow zero chunk"); + "CartesianProductCombination doesn't allow zero chunk"); this.numChunk = Arrays.copyOf(numChunk, numChunk.length); combination = new Integer[numChunk.length]; factor = new Integer[numChunk.length]; - factor[factor.length-1] = 1; - for (int i = combination.length-2; i >= 0; i--) { - factor[i] = factor[i+1]* numChunk[i+1]; + factor[factor.length - 1] = 1; + for (int i = combination.length - 2; i >= 0; i--) { + factor[i] = factor[i + 1] * numChunk[i + 1]; } } @@ -93,8 +93,8 @@ public void firstTaskWithFixedChunk(int chunkId) { public boolean nextTaskWithFixedChunk() { Preconditions.checkArgument(positionId >= 0 && positionId < combination.length); int i; - for (i = combination.length-1; i >= 0; i--) { - if (i != positionId && combination[i] != numChunk[i]-1) { + for (i = combination.length - 1; i >= 0; i--) { + if (i != positionId && combination[i] != numChunk[i] - 1) { break; } } @@ -127,8 +127,8 @@ public void firstTask() { */ public boolean nextTask() { int i; - for (i = combination.length-1; i >= 0; i--) { - if (combination[i] != numChunk[i]-1) { + for (i = combination.length - 1; i >= 0; i--) { + if (combination[i] != numChunk[i] - 1) { break; } } @@ -138,7 +138,7 @@ public boolean nextTask() { } combination[i]++; - Arrays.fill(combination, i+1, combination.length, 0); + Arrays.fill(combination, i + 1, combination.length, 0); return true; } @@ -148,7 +148,7 @@ public boolean nextTask() { public int getTaskId() { int chunkId = 0; for (int i = 0; i < combination.length; i++) { - chunkId += combination[i]*factor[i]; + chunkId += combination[i] * factor[i]; } return chunkId; } @@ -157,7 +157,7 @@ public static CartesianProductCombination fromTaskId(int[] numChunk, int taskId) { CartesianProductCombination result = new CartesianProductCombination(numChunk); for (int i = 0; i < result.combination.length; i++) { - result.combination[i] = taskId/result.factor[i]; + result.combination[i] = taskId / result.factor[i]; taskId %= result.factor[i]; } return result; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductConfig.java index e47b083127..8c0b2e7607 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductConfig.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductConfig.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -47,8 +47,8 @@ public class CartesianProductConfig { private final boolean isPartitioned; private final String[] sources; - // numPartition[i] means how many partitions sourceVertices[i] will generate - // (not used in fair cartesian product) + // numPartition[i] means how many partitions sourceVertices[i] will generate + // (not used in fair cartesian product) private final int[] numPartitions; private final CartesianProductFilterDescriptor filterDescriptor; @@ -59,7 +59,7 @@ public class CartesianProductConfig { public CartesianProductConfig(List sources) { Preconditions.checkArgument(sources != null, "source list cannot be null"); Preconditions.checkArgument(sources.size() > 1, - "there must be more than 1 source " + "67, currently only " + sources.size()); + "there must be more than 1 source " + "67, currently only " + sources.size()); this.isPartitioned = false; this.sources = sources.toArray(new String[sources.size()]); @@ -84,7 +84,7 @@ public CartesianProductConfig(Map vertexPartitionMap, CartesianProductFilterDescriptor filterDescriptor) { Preconditions.checkArgument(vertexPartitionMap != null, "vertex-partition map cannot be null"); Preconditions.checkArgument(vertexPartitionMap.size() > 1, - "there must be more than 1 source vertices, currently only " + vertexPartitionMap.size()); + "there must be more than 1 source vertices, currently only " + vertexPartitionMap.size()); this.isPartitioned = true; this.numPartitions = new int[vertexPartitionMap.size()]; @@ -113,10 +113,10 @@ protected CartesianProductConfig(int[] numPartitions, String[] sources, Preconditions.checkArgument(numPartitions != null, "partitions count array can't be null"); Preconditions.checkArgument(sources != null, "source array can't be null"); Preconditions.checkArgument(numPartitions.length == sources.length, - "partitions count array(length: " + numPartitions.length + ") and source array " + - "(length: " + sources.length + ") cannot have different length"); + "partitions count array(length: " + numPartitions.length + ") and source array " + + "(length: " + sources.length + ") cannot have different length"); Preconditions.checkArgument(sources.length > 1, - "there must be more than 1 source " + ", currently only " + sources.length); + "there must be more than 1 source " + ", currently only " + sources.length); this.isPartitioned = true; this.numPartitions = numPartitions; @@ -144,14 +144,14 @@ protected void checkNumPartitions() { boolean isUnpartitioned = true; for (int i = 0; i < numPartitions.length; i++) { Preconditions.checkArgument(this.numPartitions[i] > 0, - "Vertex " + sources[i] + "has negative (" + numPartitions[i] + ") partitions"); + "Vertex " + sources[i] + "has negative (" + numPartitions[i] + ") partitions"); isUnpartitioned = isUnpartitioned && numPartitions[i] == 1; } Preconditions.checkArgument(!isUnpartitioned, - "every source has 1 partition in a partitioned case"); + "every source has 1 partition in a partitioned case"); } else { Preconditions.checkArgument(this.numPartitions == null, - "partition counts should be null in fair cartesian product"); + "partition counts should be null in fair cartesian product"); } } @@ -187,9 +187,9 @@ public UserPayload toUserPayload(TezConfiguration conf) throws IOException { protected CartesianProductConfigProto toProto(TezConfiguration conf) { CartesianProductConfigProto.Builder builder = - CartesianProductConfigProto.newBuilder(); + CartesianProductConfigProto.newBuilder(); builder.setIsPartitioned(this.isPartitioned) - .addAllSources(Arrays.asList(sources)); + .addAllSources(Arrays.asList(sources)); if (isPartitioned) { builder.addAllNumPartitions(Ints.asList(numPartitions)); @@ -204,60 +204,60 @@ protected CartesianProductConfigProto toProto(TezConfiguration conf) { if (conf != null) { builder.setMinFraction(conf.getFloat( - CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MIN_FRACTION, - CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MIN_FRACTION_DEFAULT)); + CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MIN_FRACTION, + CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MIN_FRACTION_DEFAULT)); builder.setMaxFraction(conf.getFloat( - CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MAX_FRACTION, - CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MAX_FRACTION_DEFAULT)); + CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MAX_FRACTION, + CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MAX_FRACTION_DEFAULT)); builder.setMaxParallelism(conf.getInt( - CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MAX_PARALLELISM, - CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MAX_PARALLELISM_DEFAULT)); + CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MAX_PARALLELISM, + CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MAX_PARALLELISM_DEFAULT)); builder.setMinOpsPerWorker(conf.getLong( - CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MIN_OPS_PER_WORKER, - CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MIN_OPS_PER_WORKER_DEFAULT)); + CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MIN_OPS_PER_WORKER, + CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MIN_OPS_PER_WORKER_DEFAULT)); builder.setEnableGrouping(conf.getBoolean( - CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_ENABLE_GROUPING, - CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_ENABLE_GROUPING_DEFAULT)); + CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_ENABLE_GROUPING, + CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_ENABLE_GROUPING_DEFAULT)); if (conf.get(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_GROUPING_FRACTION) != null) { builder.setGroupingFraction(Float.parseFloat( - conf.get(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_GROUPING_FRACTION))); + conf.get(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_GROUPING_FRACTION))); Preconditions.checkArgument(0 < builder.getGroupingFraction() && - builder.getGroupingFraction() <= 1, "grouping fraction should be larger than 0 and less" + - " or equal to 1, current value: " + builder.getGroupingFraction()); + builder.getGroupingFraction() <= 1, "grouping fraction should be larger than 0 and less" + + " or equal to 1, current value: " + builder.getGroupingFraction()); } if (conf.get(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_NUM_PARTITIONS) != null) { builder.setNumPartitionsForFairCase(Integer.parseInt( - conf.get(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_NUM_PARTITIONS))); + conf.get(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_NUM_PARTITIONS))); Preconditions.checkArgument(builder.getNumPartitionsForFairCase() > 0, - "Number of partitions for fair cartesian product should be positive integer"); + "Number of partitions for fair cartesian product should be positive integer"); } } Preconditions.checkArgument(builder.getMinFraction() <= builder.getMaxFraction(), - "min fraction(" + builder.getMinFraction() + ") should be less than max fraction(" + - builder.getMaxFraction() + ") in cartesian product slow start"); + "min fraction(" + builder.getMinFraction() + ") should be less than max fraction(" + + builder.getMaxFraction() + ") in cartesian product slow start"); Preconditions.checkArgument(builder.getMaxParallelism() > 0, - "max parallelism must be positive, currently is " + builder.getMaxParallelism()); + "max parallelism must be positive, currently is " + builder.getMaxParallelism()); Preconditions.checkArgument(builder.getMinOpsPerWorker() > 0, - "Min ops per worker must be positive, currently is " + builder.getMinOpsPerWorker()); + "Min ops per worker must be positive, currently is " + builder.getMinOpsPerWorker()); return builder.build(); } protected static CartesianProductConfigProto userPayloadToProto(UserPayload payload) - throws InvalidProtocolBufferException { + throws InvalidProtocolBufferException { Preconditions.checkArgument(payload != null, "UserPayload is null"); Preconditions.checkArgument(payload.getPayload() != null, "UserPayload carreis null payload"); return - CartesianProductConfigProto.parseFrom(ByteString.copyFrom(payload.getPayload())); + CartesianProductConfigProto.parseFrom(ByteString.copyFrom(payload.getPayload())); } protected static CartesianProductConfig fromUserPayload(UserPayload payload) - throws InvalidProtocolBufferException { + throws InvalidProtocolBufferException { return fromProto(userPayloadToProto(payload)); } protected static CartesianProductConfig fromProto( - CartesianProductConfigProto proto) { + CartesianProductConfigProto proto) { if (!proto.getIsPartitioned()) { return new CartesianProductConfig(proto.getSourcesList()); } else { @@ -268,11 +268,11 @@ protected static CartesianProductConfig fromProto( filterDescriptor = new CartesianProductFilterDescriptor(proto.getFilterClassName()); if (proto.hasFilterUserPayload()) { filterDescriptor.setUserPayload( - UserPayload.create(ByteBuffer.wrap(proto.getFilterUserPayload().toByteArray()))); + UserPayload.create(ByteBuffer.wrap(proto.getFilterUserPayload().toByteArray()))); } } return new CartesianProductConfig(Ints.toArray(proto.getNumPartitionsList()), - sourceVertices, filterDescriptor); + sourceVertices, filterDescriptor); } } } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductEdgeManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductEdgeManager.java index 0a3346ea16..cb1d7cca06 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductEdgeManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductEdgeManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -43,11 +43,11 @@ public CartesianProductEdgeManager(EdgeManagerPluginContext context) { public void initialize() throws Exception { Preconditions.checkArgument(getContext().getUserPayload() != null); CartesianProductConfigProto config = CartesianProductConfigProto.parseFrom( - ByteString.copyFrom(getContext().getUserPayload().getPayload())); + ByteString.copyFrom(getContext().getUserPayload().getPayload())); // no need to check config because config comes from VM and is already checked by VM edgeManagerReal = config.getIsPartitioned() - ? new CartesianProductEdgeManagerPartitioned(getContext()) - : new FairCartesianProductEdgeManager(getContext()); + ? new CartesianProductEdgeManagerPartitioned(getContext()) + : new FairCartesianProductEdgeManager(getContext()); edgeManagerReal.initialize(config); } @@ -71,7 +71,7 @@ public int routeInputErrorEventToSource(int destTaskId, int failedInputId) throw public EventRouteMetadata routeDataMovementEventToDestination(int srcTaskId, int srcOutputId, int destTaskId) - throws Exception { + throws Exception { return edgeManagerReal.routeDataMovementEventToDestination(srcTaskId, srcOutputId, destTaskId); } @@ -79,7 +79,7 @@ public EventRouteMetadata routeDataMovementEventToDestination(int srcTaskId, @Override public CompositeEventRouteMetadata routeCompositeDataMovementEventToDestination(int srcTaskId, int destTaskId) - throws Exception { + throws Exception { return edgeManagerReal.routeCompositeDataMovementEventToDestination(srcTaskId, destTaskId); } @@ -87,7 +87,7 @@ public CompositeEventRouteMetadata routeCompositeDataMovementEventToDestination( @Override public EventRouteMetadata routeInputSourceTaskFailedEventToDestination(int srcTaskId, int destTaskId) - throws Exception { + throws Exception { return edgeManagerReal.routeInputSourceTaskFailedEventToDestination(srcTaskId, destTaskId); } @@ -105,4 +105,4 @@ public int getNumSourceTaskPhysicalOutputs(int srcTaskId) { public int getNumDestinationConsumerTasks(int sourceTaskIndex) { return edgeManagerReal.getNumDestinationConsumerTasks(sourceTaskIndex); } -} \ No newline at end of file +} diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductEdgeManagerPartitioned.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductEdgeManagerPartitioned.java index 5f2910aa30..c3508e9c61 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductEdgeManagerPartitioned.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductEdgeManagerPartitioned.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -53,10 +53,10 @@ public void initialize(CartesianProductConfigProto config) throws Exception { if (config.hasFilterClassName()) { UserPayload userPayload = config.hasFilterUserPayload() - ? UserPayload.create(ByteBuffer.wrap(config.getFilterUserPayload().toByteArray())) : null; + ? UserPayload.create(ByteBuffer.wrap(config.getFilterUserPayload().toByteArray())) : null; try { filter = ReflectionUtils.createClazzInstance(config.getFilterClassName(), - new Class[]{UserPayload.class}, new UserPayload[]{userPayload}); + new Class[]{UserPayload.class}, new UserPayload[]{userPayload}); } catch (TezReflectionException e) { throw e; } @@ -73,18 +73,18 @@ public int routeInputErrorEventToSource(int destTaskId, int failedInputId) throw public EventRouteMetadata routeDataMovementEventToDestination(int srcTaskId, int srcOutputId, int destTaskId) throws Exception { int partition = CartesianProductCombination.fromTaskId(numPartitions, - getIdealTaskId(destTaskId)).getCombination().get(positionId); + getIdealTaskId(destTaskId)).getCombination().get(positionId); return srcOutputId != partition ? null : - EventRouteMetadata.create(1, new int[]{srcTaskId}); + EventRouteMetadata.create(1, new int[]{srcTaskId}); } @Nullable @Override public CompositeEventRouteMetadata routeCompositeDataMovementEventToDestination(int srcTaskId, - int destTaskId) - throws Exception { + int destTaskId) + throws Exception { int partition = CartesianProductCombination.fromTaskId(numPartitions, - getIdealTaskId(destTaskId)).getCombination().get(positionId); + getIdealTaskId(destTaskId)).getCombination().get(positionId); return CompositeEventRouteMetadata.create(1, srcTaskId, partition); } @@ -92,7 +92,7 @@ public CompositeEventRouteMetadata routeCompositeDataMovementEventToDestination( @Override public EventRouteMetadata routeInputSourceTaskFailedEventToDestination(int srcTaskId, int destTaskId) - throws Exception { + throws Exception { return EventRouteMetadata.create(1, new int[]{srcTaskId}); } @@ -115,7 +115,7 @@ private void generateTaskIdMapping() { List idealTaskId = new ArrayList<>(); Map vertexPartitionMap = new HashMap<>(); CartesianProductCombination combination = - new CartesianProductCombination(numPartitions); + new CartesianProductCombination(numPartitions); combination.firstTask(); do { for (int i = 0; i < sources.size(); i++) { @@ -131,4 +131,4 @@ private void generateTaskIdMapping() { private int getIdealTaskId(int realTaskId) { return taskIdMapping[realTaskId]; } -} \ No newline at end of file +} diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductEdgeManagerReal.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductEdgeManagerReal.java index 0b91ec2634..9bbb1a10fa 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductEdgeManagerReal.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductEdgeManagerReal.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -41,20 +41,20 @@ public EdgeManagerPluginContext getContext() { public void prepareForRouting() throws Exception {} public abstract int routeInputErrorEventToSource(int destTaskId, int failedInputId) - throws Exception; + throws Exception; public abstract EventRouteMetadata routeDataMovementEventToDestination(int srcTaskId, int srcOutputId, int destTaskId) - throws Exception; + throws Exception; public abstract CompositeEventRouteMetadata routeCompositeDataMovementEventToDestination(int srcTaskId, int destTaskId) - throws Exception; + throws Exception; public abstract EventRouteMetadata routeInputSourceTaskFailedEventToDestination(int srcTaskId, int destTaskId) - throws Exception; + throws Exception; public abstract int getNumDestinationTaskPhysicalInputs(int destTaskId); diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductFilter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductFilter.java index 5b6456eb57..95da684308 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductFilter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductFilter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductFilterDescriptor.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductFilterDescriptor.java index bc8175577b..93dcb1586a 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductFilterDescriptor.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductFilterDescriptor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,7 +20,7 @@ import org.apache.tez.dag.api.EntityDescriptor; public class CartesianProductFilterDescriptor - extends EntityDescriptor { + extends EntityDescriptor { public CartesianProductFilterDescriptor(String filterClassName) { super(filterClassName); diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductVertexManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductVertexManager.java index d1fcece722..bf0d2ab5c7 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductVertexManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductVertexManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -63,14 +63,14 @@ public class CartesianProductVertexManager extends VertexManagerPlugin { * this value */ public static final String TEZ_CARTESIAN_PRODUCT_SLOW_START_MIN_FRACTION = - "tez.cartesian-product.min-src-fraction"; + "tez.cartesian-product.min-src-fraction"; public static final float TEZ_CARTESIAN_PRODUCT_SLOW_START_MIN_FRACTION_DEFAULT = 0.25f; /** * Schedule all tasks when the fraction of finished cartesian product source tasks reach this value */ public static final String TEZ_CARTESIAN_PRODUCT_SLOW_START_MAX_FRACTION = - "tez.cartesian-product.max-src-fraction"; + "tez.cartesian-product.max-src-fraction"; public static final float TEZ_CARTESIAN_PRODUCT_SLOW_START_MAX_FRACTION_DEFAULT = 0.75f; /** @@ -78,7 +78,7 @@ public class CartesianProductVertexManager extends VertexManagerPlugin { * Set this if auto determined num partition is not large enough */ public static final String TEZ_CARTESIAN_PRODUCT_NUM_PARTITIONS = - "tez.cartesian-product.num-partitions"; + "tez.cartesian-product.num-partitions"; /** * Whether to disable grouping in fair cartesian product @@ -86,7 +86,7 @@ public class CartesianProductVertexManager extends VertexManagerPlugin { * unnecessary overhead caused by multiple partitions. */ public static final String TEZ_CARTESIAN_PRODUCT_ENABLE_GROUPING = - "tez.cartesian-product.disable-grouping"; + "tez.cartesian-product.disable-grouping"; public static final boolean TEZ_CARTESIAN_PRODUCT_ENABLE_GROUPING_DEFAULT = true; /** @@ -97,14 +97,14 @@ public class CartesianProductVertexManager extends VertexManagerPlugin { * If not set, auto grouping will begin once every source vertex generate enough output */ public static final String TEZ_CARTESIAN_PRODUCT_GROUPING_FRACTION = - "tez.cartesian-product.grouping-fraction"; + "tez.cartesian-product.grouping-fraction"; /** * Max parallelism, for fair cartesian product only. * This is used to avoid get too many tasks. The value must be positive. */ public static final String TEZ_CARTESIAN_PRODUCT_MAX_PARALLELISM = - "tez.cartesian-product.max-parallelism"; + "tez.cartesian-product.max-parallelism"; public static final int TEZ_CARTESIAN_PRODUCT_MAX_PARALLELISM_DEFAULT = 1000; /** @@ -112,7 +112,7 @@ public class CartesianProductVertexManager extends VertexManagerPlugin { * This is used to avoid a task gets too small workload. The value must be positive. */ public static final String TEZ_CARTESIAN_PRODUCT_MIN_OPS_PER_WORKER = - "tez.cartesian-product.min-ops-per-worker"; + "tez.cartesian-product.min-ops-per-worker"; public static final long TEZ_CARTESIAN_PRODUCT_MIN_OPS_PER_WORKER_DEFAULT = 1000000; private CartesianProductVertexManagerReal vertexManagerReal = null; @@ -120,13 +120,13 @@ public class CartesianProductVertexManager extends VertexManagerPlugin { public CartesianProductVertexManager(VertexManagerPluginContext context) { super(context); Preconditions.checkArgument(context.getVertexNumTasks(context.getVertexName()) == -1, - "Vertex with CartesianProductVertexManager cannot use pre-defined parallelism"); + "Vertex with CartesianProductVertexManager cannot use pre-defined parallelism"); } @Override public void initialize() throws Exception { CartesianProductConfigProto config = CartesianProductConfigProto.parseFrom( - ByteString.copyFrom(getContext().getUserPayload().getPayload())); + ByteString.copyFrom(getContext().getUserPayload().getPayload())); // check whether DAG and config are is consistent Map edgePropertyMap = getContext().getInputVertexEdgeProperties(); Set sourceVerticesDAG = edgePropertyMap.keySet(); @@ -146,46 +146,46 @@ public void initialize() throws Exception { EdgeProperty edgeProperty = entry.getValue(); EdgeManagerPluginDescriptor empDescriptor = edgeProperty.getEdgeManagerDescriptor(); if (empDescriptor != null - && empDescriptor.getClassName().equals(CartesianProductEdgeManager.class.getName())) { + && empDescriptor.getClassName().equals(CartesianProductEdgeManager.class.getName())) { Preconditions.checkArgument( - sourceVerticesConfig.contains(vertex) || sourceVerticesConfig.contains(group), - vertex + " has CartesianProductEdgeManager but isn't in " + - "CartesianProductVertexManagerConfig"); + sourceVerticesConfig.contains(vertex) || sourceVerticesConfig.contains(group), + vertex + " has CartesianProductEdgeManager but isn't in " + + "CartesianProductVertexManagerConfig"); } else { Preconditions.checkArgument( - !sourceVerticesConfig.contains(vertex) && !sourceVerticesConfig.contains(group), - vertex + " has no CartesianProductEdgeManager but is in " + - "CartesianProductVertexManagerConfig"); + !sourceVerticesConfig.contains(vertex) && !sourceVerticesConfig.contains(group), + vertex + " has no CartesianProductEdgeManager but is in " + + "CartesianProductVertexManagerConfig"); } if (edgeProperty.getDataMovementType() == CUSTOM) { Preconditions.checkArgument( - sourceVerticesConfig.contains(vertex) || sourceVerticesConfig.contains(group), - "Only broadcast and cartesian product edges are allowed in cartesian product vertex"); + sourceVerticesConfig.contains(vertex) || sourceVerticesConfig.contains(group), + "Only broadcast and cartesian product edges are allowed in cartesian product vertex"); } else { Preconditions.checkArgument(edgeProperty.getDataMovementType() == BROADCAST, - "Only broadcast and cartesian product edges are allowed in cartesian product vertex"); + "Only broadcast and cartesian product edges are allowed in cartesian product vertex"); } } for (String src : sourceVerticesConfig) { List vertices = - vertexGroups.containsKey(src) ? vertexGroups.get(src) : Collections.singletonList(src); + vertexGroups.containsKey(src) ? vertexGroups.get(src) : Collections.singletonList(src); for (String v : vertices) { Preconditions.checkArgument( - sourceVerticesDAG.contains(v), - v + " is in CartesianProductVertexManagerConfig but not a source vertex in DAG"); + sourceVerticesDAG.contains(v), + v + " is in CartesianProductVertexManagerConfig but not a source vertex in DAG"); Preconditions.checkArgument( - edgePropertyMap.get(v).getEdgeManagerDescriptor().getClassName() - .equals(CartesianProductEdgeManager.class.getName()), - v + " is in CartesianProductVertexManagerConfig and a source vertex, but has no " + - "CartesianProductEdgeManager"); + edgePropertyMap.get(v).getEdgeManagerDescriptor().getClassName() + .equals(CartesianProductEdgeManager.class.getName()), + v + " is in CartesianProductVertexManagerConfig and a source vertex, but has no " + + "CartesianProductEdgeManager"); } } vertexManagerReal = config.getIsPartitioned() - ? new CartesianProductVertexManagerPartitioned(getContext()) - : new FairCartesianProductVertexManager(getContext()); + ? new CartesianProductVertexManagerPartitioned(getContext()) + : new FairCartesianProductVertexManager(getContext()); vertexManagerReal.initialize(config); } @@ -224,7 +224,7 @@ public void onVertexStarted(List completions) throws Exce } @Override - public void onVertexStateUpdated(VertexStateUpdate stateUpdate) throws Exception{ + public void onVertexStateUpdated(VertexStateUpdate stateUpdate) throws Exception { vertexManagerReal.onVertexStateUpdated(stateUpdate); } @@ -232,4 +232,4 @@ public void onVertexStateUpdated(VertexStateUpdate stateUpdate) throws Exception public void onSourceTaskCompleted(TaskAttemptIdentifier attempt) throws Exception { vertexManagerReal.onSourceTaskCompleted(attempt); } -} \ No newline at end of file +} diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductVertexManagerPartitioned.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductVertexManagerPartitioned.java index e4aaad6476..7d8ea0f0b1 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductVertexManagerPartitioned.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductVertexManagerPartitioned.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -60,7 +60,7 @@ class CartesianProductVertexManagerPartitioned extends CartesianProductVertexMan private int totalNumSrcTasks = 0; private int lastScheduledTaskId = -1; private static final Logger LOG = - LoggerFactory.getLogger(CartesianProductVertexManagerPartitioned.class); + LoggerFactory.getLogger(CartesianProductVertexManagerPartitioned.class); public CartesianProductVertexManagerPartitioned(VertexManagerPluginContext context) { super(context); @@ -71,16 +71,16 @@ public void initialize(CartesianProductConfigProto config) throws TezReflectionE this.sourceVertices = config.getSourcesList(); this.numPartitions = Ints.toArray(config.getNumPartitionsList()); this.minFraction = config.hasMinFraction() ? config.getMinFraction() - : CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MIN_FRACTION_DEFAULT; + : CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MIN_FRACTION_DEFAULT; this.maxFraction = config.hasMaxFraction() ? config.getMaxFraction() - : CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MAX_FRACTION_DEFAULT; + : CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_SLOW_START_MAX_FRACTION_DEFAULT; if (config.hasFilterClassName()) { UserPayload userPayload = config.hasFilterUserPayload() - ? UserPayload.create(ByteBuffer.wrap(config.getFilterUserPayload().toByteArray())) : null; + ? UserPayload.create(ByteBuffer.wrap(config.getFilterUserPayload().toByteArray())) : null; try { filter = ReflectionUtils.createClazzInstance(config.getFilterClassName(), - new Class[]{UserPayload.class}, new UserPayload[]{userPayload}); + new Class[]{UserPayload.class}, new UserPayload[]{userPayload}); } catch (TezReflectionException e) { LOG.error("Creating filter failed"); throw e; @@ -107,7 +107,7 @@ public void onVertexManagerEventReceived(VertexManagerEvent vmEvent) throws IOEx @Override public synchronized void onVertexStarted(List completions) - throws Exception { + throws Exception { vertexStarted = true; if (completions != null) { for (TaskAttemptIdentifier attempt : completions) { @@ -119,7 +119,7 @@ public synchronized void onVertexStarted(List completions } @Override - public synchronized void onVertexStateUpdated(VertexStateUpdate stateUpdate) throws IOException{ + public synchronized void onVertexStateUpdated(VertexStateUpdate stateUpdate) throws IOException { VertexState state = stateUpdate.getVertexState(); if (state == VertexState.CONFIGURED) { @@ -128,7 +128,7 @@ public synchronized void onVertexStateUpdated(VertexStateUpdate stateUpdate) thr } numCPSrcNotInConfiguredState--; totalNumSrcTasks += getContext().getVertexNumTasks(stateUpdate.getVertexName()); - } else if (state == VertexState.RUNNING){ + } else if (state == VertexState.RUNNING) { numBroadcastSrcNotInRunningState--; } // try schedule because there may be no more vertex start and source completions @@ -157,7 +157,7 @@ private void reconfigureVertex() throws IOException { Map vertexPartitionMap = new HashMap<>(); CartesianProductCombination combination = - new CartesianProductCombination(numPartitions); + new CartesianProductCombination(numPartitions); combination.firstTask(); do { for (int i = 0; i < sourceVertices.size(); i++) { @@ -182,25 +182,25 @@ private void tryScheduleTask() { return; } // determine the destination task with largest id to schedule - float percentFinishedSrcTask = numFinishedSrcTasks*1f/totalNumSrcTasks; + float percentFinishedSrcTask = numFinishedSrcTasks * 1f / totalNumSrcTasks; int numTaskToSchedule; if (percentFinishedSrcTask < minFraction) { numTaskToSchedule = 0; } else if (minFraction <= percentFinishedSrcTask && percentFinishedSrcTask <= maxFraction) { numTaskToSchedule = (int) ((percentFinishedSrcTask - minFraction) - /(maxFraction - minFraction) * parallelism); + / (maxFraction - minFraction) * parallelism); } else { numTaskToSchedule = parallelism; } // schedule tasks if there are more we can schedule - if (numTaskToSchedule-1 > lastScheduledTaskId) { + if (numTaskToSchedule - 1 > lastScheduledTaskId) { List scheduleTaskRequests = new ArrayList<>(); for (int i = lastScheduledTaskId + 1; i < numTaskToSchedule; i++) { scheduleTaskRequests.add(ScheduleTaskRequest.create(i, null)); } - lastScheduledTaskId = numTaskToSchedule-1; + lastScheduledTaskId = numTaskToSchedule - 1; getContext().scheduleTasks(scheduleTaskRequests); } } -} \ No newline at end of file +} diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductVertexManagerReal.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductVertexManagerReal.java index f28f4a374d..c29ad4586c 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductVertexManagerReal.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/CartesianProductVertexManagerReal.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/FairCartesianProductEdgeManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/FairCartesianProductEdgeManager.java index 3085e5e29c..5395179313 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/FairCartesianProductEdgeManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/FairCartesianProductEdgeManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -28,7 +28,6 @@ import static org.apache.tez.runtime.library.cartesianproduct.CartesianProductCombination.fromTaskId; import static org.apache.tez.runtime.library.cartesianproduct.CartesianProductUserPayload.*; - class FairCartesianProductEdgeManager extends CartesianProductEdgeManagerReal { private int numPartition; // position of current source in all cartesian product sources @@ -64,7 +63,7 @@ public void initialize(CartesianProductConfigProto config) { // initialize after reconfiguration this.numChunkPerSrc = Ints.toArray(config.getNumChunksList()); grouper.init(getContext().getSourceVertexNumTasks() * numPartition, - numChunkPerSrc[positionInSrc]); + numChunkPerSrc[positionInSrc]); this.numTaskPerSrcVertexInGroup = Ints.toArray(config.getNumTaskPerVertexInGroupList()); this.positionInGroup = config.getPositionInGroup(); @@ -90,7 +89,7 @@ public EventRouteMetadata routeDataMovementEventToDestination(int srcTaskId, int int chunkId = fromTaskId(numChunkPerSrc, destTaskId).getCombination().get(positionInSrc); if (grouper.isInGroup(itemId, chunkId)) { int idx = itemId - grouper.getFirstItemInGroup(chunkId) + getItemIdOffset(chunkId); - return EventRouteMetadata.create(1, new int[] {idx}); + return EventRouteMetadata.create(1, new int[]{idx}); } return null; } @@ -99,17 +98,17 @@ public EventRouteMetadata routeDataMovementEventToDestination(int srcTaskId, int @Override public CompositeEventRouteMetadata routeCompositeDataMovementEventToDestination(int srcTaskId, int destTaskId) - throws Exception { + throws Exception { int chunkId = fromTaskId(numChunkPerSrc, destTaskId).getCombination().get(positionInSrc); int firstItemInChunk = grouper.getFirstItemInGroup(chunkId); int lastItemInChunk = grouper.getLastItemInGroup(chunkId); - int firstItemInSrcTask = srcTaskId * numPartition; + int firstItemInSrcTask = srcTaskId * numPartition; int lastItemInSrcTask = firstItemInSrcTask + numPartition - 1; if (!(lastItemInChunk < firstItemInSrcTask || firstItemInChunk > lastItemInSrcTask)) { int firstItem = Math.max(firstItemInChunk, firstItemInSrcTask); int lastItem = Math.min(lastItemInChunk, lastItemInSrcTask); return CompositeEventRouteMetadata.create(lastItem - firstItem + 1, - firstItem - firstItemInChunk + getItemIdOffset(chunkId), firstItem - firstItemInSrcTask); + firstItem - firstItemInChunk + getItemIdOffset(chunkId), firstItem - firstItemInSrcTask); } return null; } @@ -123,7 +122,7 @@ private int getItemIdOffset(int chunkId) { int offset = 0; for (int i = 0; i < positionInGroup; i++) { grouperForComputeOffset.init(numTaskPerSrcVertexInGroup[i] * numPartition, - numChunkPerSrc[positionInSrc]); + numChunkPerSrc[positionInSrc]); offset += grouperForComputeOffset.getNumItemsInGroup(chunkId); } return offset; @@ -133,7 +132,7 @@ private int getItemIdOffset(int chunkId) { @Override public EventRouteMetadata routeInputSourceTaskFailedEventToDestination(int srcTaskId, int destTaskId) - throws Exception { + throws Exception { int chunkId = fromTaskId(numChunkPerSrc, destTaskId).getCombination().get(positionInSrc); int firstItemInChunk = grouper.getFirstItemInGroup(chunkId); int lastItemInChunk = grouper.getLastItemInGroup(chunkId); @@ -168,7 +167,7 @@ public int getNumSourceTaskPhysicalOutputs(int srcTaskId) { @Override public int getNumDestinationConsumerTasks(int sourceTaskIndex) { int numChunk = grouper.getGroupId(sourceTaskIndex * numPartition + numPartition - 1) - - grouper.getGroupId(sourceTaskIndex * numPartition) + 1; + - grouper.getGroupId(sourceTaskIndex * numPartition) + 1; return numDestConsumerPerChunk * numChunk; } -} \ No newline at end of file +} diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/FairCartesianProductVertexManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/FairCartesianProductVertexManager.java index 7b519e4400..215da9c586 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/FairCartesianProductVertexManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/cartesianproduct/FairCartesianProductVertexManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -98,7 +98,6 @@ public String toString() { sb.append(", "); sb.append("name "); sb.append(name); - } sb.append(", num chunk ").append(numChunk); sb.append(": {"); @@ -198,7 +197,7 @@ public boolean isChunkCompleted(int chunkId) { } private static final Logger LOG = - org.slf4j.LoggerFactory.getLogger(FairCartesianProductVertexManager.class); + org.slf4j.LoggerFactory.getLogger(FairCartesianProductVertexManager.class); private CartesianProductConfigProto config; private List sourceList; @@ -233,11 +232,11 @@ public FairCartesianProductVertexManager(VertexManagerPluginContext context) { public void initialize(CartesianProductConfigProto config) throws Exception { this.config = config; maxParallelism = config.hasMaxParallelism() ? config.getMaxParallelism() - :CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MAX_PARALLELISM_DEFAULT; + : CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MAX_PARALLELISM_DEFAULT; enableGrouping = config.hasEnableGrouping() ? config.getEnableGrouping() - :CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_ENABLE_GROUPING_DEFAULT; + : CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_ENABLE_GROUPING_DEFAULT; minOpsPerWorker = config.hasMinOpsPerWorker() ? config.getMinOpsPerWorker() - : CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MIN_OPS_PER_WORKER_DEFAULT; + : CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MIN_OPS_PER_WORKER_DEFAULT; sourceList = config.getSourcesList(); if (config.hasNumPartitionsForFairCase()) { numPartitions = config.getNumPartitionsForFairCase(); @@ -247,8 +246,8 @@ public void initialize(CartesianProductConfigProto config) throws Exception { for (Map.Entry e : getContext().getInputVertexEdgeProperties().entrySet()) { if (e.getValue().getDataMovementType() == CUSTOM - && e.getValue().getEdgeManagerDescriptor().getClassName() - .equals(CartesianProductEdgeManager.class.getName())) { + && e.getValue().getEdgeManagerDescriptor().getClassName() + .equals(CartesianProductEdgeManager.class.getName())) { srcVerticesByName.put(e.getKey(), new SrcVertex()); srcVerticesByName.get(e.getKey()).name = e.getKey(); getContext().registerForVertexStateUpdates(e.getKey(), EnumSet.of(VertexState.CONFIGURED)); @@ -279,7 +278,7 @@ public void initialize(CartesianProductConfigProto config) throws Exception { } minNumRecordForEstimation = - (long) Math.pow(minOpsPerWorker * maxParallelism, 1.0 / sourceList.size()); + (long) Math.pow(minOpsPerWorker * maxParallelism, 1.0 / sourceList.size()); numChunksPerSrc = new int[sourcesByName.size()]; getContext().vertexReconfigurationPlanned(); @@ -287,7 +286,7 @@ public void initialize(CartesianProductConfigProto config) throws Exception { @Override public synchronized void onVertexStarted(List completions) - throws Exception { + throws Exception { vertexStarted = true; if (completions != null) { LOG.info("OnVertexStarted with " + completions.size() + " completed source task"); @@ -330,13 +329,13 @@ private void addCompletedSrcTaskToProcess(TaskAttemptIdentifier attempt) { private boolean tryStartSchedule() { vertexStartSchedule = - (vertexReconfigured && vertexStarted && numBroadcastSrcNotInRunningState == 0); + (vertexReconfigured && vertexStarted && numBroadcastSrcNotInRunningState == 0); return vertexStartSchedule; } @Override public synchronized void onVertexManagerEventReceived(VertexManagerEvent vmEvent) - throws IOException { + throws IOException { /* vmEvent after reconfigure doesn't matter */ if (vertexReconfigured) { return; @@ -344,7 +343,7 @@ public synchronized void onVertexManagerEventReceived(VertexManagerEvent vmEvent if (vmEvent.getUserPayload() != null) { String srcVertex = - vmEvent.getProducerAttemptIdentifier().getTaskIdentifier().getVertexIdentifier().getName(); + vmEvent.getProducerAttemptIdentifier().getTaskIdentifier().getVertexIdentifier().getName(); SrcVertex srcV = srcVerticesByName.get(srcVertex); // vmEvent from non-cp vertex doesn't matter @@ -353,10 +352,10 @@ public synchronized void onVertexManagerEventReceived(VertexManagerEvent vmEvent } VertexManagerEventPayloadProto proto = - VertexManagerEventPayloadProto.parseFrom(ByteString.copyFrom(vmEvent.getUserPayload())); + VertexManagerEventPayloadProto.parseFrom(ByteString.copyFrom(vmEvent.getUserPayload())); srcV.numRecord += proto.getNumRecord(); srcV.taskWithVMEvent.add( - vmEvent.getProducerAttemptIdentifier().getTaskIdentifier().getIdentifier()); + vmEvent.getProducerAttemptIdentifier().getTaskIdentifier().getIdentifier()); } tryScheduleTasks(); @@ -385,8 +384,8 @@ private boolean tryReconfigure() throws IOException { // every src vertex must complete a certain number of task before we do estimation for (SrcVertex srcV : srcVerticesByName.values()) { if (srcV.taskCompleted.getCardinality() < srcV.numTask - && (srcV.numTask * config.getGroupingFraction() > srcV.taskCompleted.getCardinality() - || srcV.numRecord == 0)) { + && (srcV.numTask * config.getGroupingFraction() > srcV.taskCompleted.getCardinality() + || srcV.numRecord == 0)) { return false; } } @@ -395,16 +394,16 @@ private boolean tryReconfigure() throws IOException { // or all its tasks already finish but we cannot get enough result for estimation for (SrcVertex srcV : srcVerticesByName.values()) { if (srcV.numRecord < minNumRecordForEstimation - && srcV.taskWithVMEvent.getCardinality() < srcV.numTask) { + && srcV.taskWithVMEvent.getCardinality() < srcV.numTask) { return false; } } } LOG.info("Start reconfiguring vertex " + getContext().getVertexName() - + ", max parallelism: " + maxParallelism - + ", min-ops-per-worker: " + minOpsPerWorker - + ", num partition: " + numPartitions); + + ", max parallelism: " + maxParallelism + + ", min-ops-per-worker: " + minOpsPerWorker + + ", num partition: " + numPartitions); for (Source src : sourcesByName.values()) { LOG.info(src.toString()); } @@ -419,7 +418,7 @@ private boolean tryReconfigure() throws IOException { } try { - totalOps = LongMath.checkedMultiply(totalOps, src.numRecord); + totalOps = LongMath.checkedMultiply(totalOps, src.numRecord); } catch (ArithmeticException e) { LOG.info("totalOps exceeds " + Long.MAX_VALUE + ", capping to " + Long.MAX_VALUE); totalOps = Long.MAX_VALUE; @@ -476,7 +475,7 @@ private boolean tryReconfigure() throws IOException { SrcVertex srcV = src.srcVertices.get(i); builder.setPositionInGroup(i); edgeProperties.get(srcV.name).getEdgeManagerDescriptor() - .setUserPayload(UserPayload.create(ByteBuffer.wrap(builder.build().toByteArray()))); + .setUserPayload(UserPayload.create(ByteBuffer.wrap(builder.build().toByteArray()))); builder.addNumTaskPerVertexInGroup(srcV.numTask); } } @@ -517,8 +516,8 @@ private void determineNumChunks(Map sourcesByName, int paralleli for (Source src : sourcesByName.values()) { if (src.numChunk != 1) { src.numChunk = Math.min(maxParallelism, - Math.min(src.getSrcVertexWithMostOutput().numTask * numPartitions, - Math.max(1, (int) (src.numRecord * k)))); + Math.min(src.getSrcVertexWithMostOutput().numTask * numPartitions, + Math.max(1, (int) (src.numRecord * k)))); } } } @@ -548,7 +547,7 @@ private void scheduleTasksDependOnCompletion(TaskAttemptIdentifier attempt) { List requests = new ArrayList<>(); CartesianProductCombination combination = - new CartesianProductCombination(numChunksPerSrc, src.position); + new CartesianProductCombination(numChunksPerSrc, src.position); grouper.init(srcV.numTask * numPartitions, src.numChunk); int firstRelevantChunk = grouper.getGroupId(taskId * numPartitions); @@ -565,9 +564,9 @@ private void scheduleTasksDependOnCompletion(TaskAttemptIdentifier attempt) { // a task is ready for schedule only if all its src chunk has been completed boolean readyToSchedule = src.isChunkCompleted(list.get(src.position)); for (int srcId = 0; readyToSchedule && srcId < list.size(); srcId++) { - if (srcId != src.position){ + if (srcId != src.position) { readyToSchedule = - sourcesByName.get(sourceList.get(srcId)).isChunkCompleted(list.get(srcId)); + sourcesByName.get(sourceList.get(srcId)).isChunkCompleted(list.get(srcId)); } } @@ -582,4 +581,4 @@ private void scheduleTasksDependOnCompletion(TaskAttemptIdentifier attempt) { getContext().scheduleTasks(requests); } } -} \ No newline at end of file +} diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/CompositeInputAttemptIdentifier.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/CompositeInputAttemptIdentifier.java index 30295bd399..de6990d98c 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/CompositeInputAttemptIdentifier.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/CompositeInputAttemptIdentifier.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,12 +36,11 @@ public CompositeInputAttemptIdentifier(int inputIdentifier, int attemptNumber, S } public CompositeInputAttemptIdentifier(int inputIdentifier, int attemptNumber, String pathComponent, - boolean shared, SPILL_INFO fetchTypeInfo, int spillEventId, int inputIdentifierCount) { + boolean shared, SPILL_INFO fetchTypeInfo, int spillEventId, int inputIdentifierCount) { super(inputIdentifier, attemptNumber, pathComponent, shared, fetchTypeInfo, spillEventId); this.inputIdentifierCount = inputIdentifierCount; } - public int getInputIdentifierCount() { return inputIdentifierCount; } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ConfigUtils.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ConfigUtils.java index 74e74f2780..2d04dd48cc 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ConfigUtils.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ConfigUtils.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.runtime.library.common; @@ -57,7 +57,7 @@ public static Class getIntermediateOutputCompressorC } // TODO Move defaults over to a constants file. - + public static boolean shouldCompressIntermediateOutput(Configuration conf) { return conf.getBoolean( TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, false); @@ -69,7 +69,7 @@ public static Class getIntermediateOutputValueClass(Configuration conf) { Object.class); return retv; } - + public static Class getIntermediateInputValueClass(Configuration conf) { Class retv = (Class) conf.getClass( TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, null, @@ -111,8 +111,6 @@ public static RawComparator getIntermediateInputKeyComparator(Configurati WritableComparable.class), conf); } - - // TODO Fix name public static RawComparator getInputKeySecondaryGroupingComparator( Configuration conf) { @@ -126,7 +124,7 @@ public static RawComparator getInputKeySecondaryGroupingComparator( return ReflectionUtils.newInstance(theClass, conf); } - + public static boolean useNewApi(Configuration conf) { return conf.getBoolean("mapred.mapper.new-api", false); } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/Constants.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/Constants.java index 81921b2e85..1f3cdfdae6 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/Constants.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/Constants.java @@ -5,9 +5,9 @@ * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

    * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -23,7 +23,7 @@ public class Constants { // TODO NEWTEZ Check which of these constants are expecting specific pieces of information which are being removed - like taskAttemptId - + public static final String TEZ = "tez"; public static final String DAG_PREFIX = "dag_"; @@ -37,7 +37,7 @@ public class Constants { public static final long DEFAULT_COMBINE_RECORDS_BEFORE_PROGRESS = 10000; // TODO NEWTEZ Remove this constant once the old code is removed. - public static final String TEZ_RUNTIME_TASK_ATTEMPT_ID = + public static final String TEZ_RUNTIME_TASK_ATTEMPT_ID = "tez.runtime.task.attempt.id"; public static final String TEZ_RUNTIME_TASK_OUTPUT_FILENAME_STRING = @@ -47,7 +47,7 @@ public class Constants { ".index"; public static final String TEZ_RUNTIME_TASK_INPUT_FILE_FORMAT_STRING = - "%s/task_%d.out"; + "%s/task_%d.out"; public static final String TEZ_RUNTIME_JOB_CREDENTIALS = "tez.runtime.job.credentials"; @@ -58,10 +58,9 @@ public class Constants { @Private public static final String TEZ_RUNTIME_TASK_MEMORY = "tez.runtime.task.memory"; - + public static final String TEZ_RUNTIME_TASK_OUTPUT_DIR = "output"; - - public static final String TEZ_RUNTIME_TASK_OUTPUT_MANAGER = - "tez.runtime.task.local.output.manager"; + public static final String TEZ_RUNTIME_TASK_OUTPUT_MANAGER = + "tez.runtime.task.local.output.manager"; } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/InputAttemptIdentifier.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/InputAttemptIdentifier.java index 16172e1daf..e7cbe740ae 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/InputAttemptIdentifier.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/InputAttemptIdentifier.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -61,12 +61,12 @@ public InputAttemptIdentifier(int inputIdentifier, int attemptNumber, String pat } public InputAttemptIdentifier(int inputIdentifier, int attemptNumber, String pathComponent, - boolean shared, SPILL_INFO fetchTypeInfo, int spillEventId) { + boolean shared, SPILL_INFO fetchTypeInfo, int spillEventId) { this.inputIdentifier = inputIdentifier; this.attemptNumber = attemptNumber; this.pathComponent = pathComponent; this.shared = shared; - this.fetchTypeInfo = (byte)fetchTypeInfo.ordinal(); + this.fetchTypeInfo = (byte) fetchTypeInfo.ordinal(); this.spillEventId = spillEventId; if (pathComponent != null && !pathComponent.startsWith(PATH_PREFIX)) { throw new TezUncheckedException( @@ -81,7 +81,7 @@ public int getInputIdentifier() { public int getAttemptNumber() { return attemptNumber; } - + public String getPathComponent() { return pathComponent; } @@ -139,6 +139,6 @@ public boolean equals(Object obj) { public String toString() { return "InputAttemptIdentifier [inputIdentifier=" + inputIdentifier + ", attemptNumber=" + attemptNumber + ", pathComponent=" - + pathComponent + ", spillType=" + fetchTypeInfo + ", spillId=" + spillEventId +"]"; + + pathComponent + ", spillType=" + fetchTypeInfo + ", spillId=" + spillEventId + "]"; } } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/InputIdentifier.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/InputIdentifier.java index 095c03f7f4..c1090b779d 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/InputIdentifier.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/InputIdentifier.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,7 +24,7 @@ public class InputIdentifier { private final int inputIndex; - + public InputIdentifier(int srcInputIndex) { this.inputIndex = srcInputIndex; } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/MemoryUpdateCallbackHandler.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/MemoryUpdateCallbackHandler.java index ff31b49984..d0c60f7698 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/MemoryUpdateCallbackHandler.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/MemoryUpdateCallbackHandler.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -44,4 +44,4 @@ public synchronized long getMemoryAssigned() { public synchronized void validateUpdateReceived() { Preconditions.checkState(updated == true, "Iniital memory update not received"); } -} \ No newline at end of file +} diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/TezRuntimeUtils.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/TezRuntimeUtils.java index a75925c5fe..6a04209115 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/TezRuntimeUtils.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/TezRuntimeUtils.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.runtime.library.common; @@ -60,13 +60,13 @@ public static String getTaskIdentifier(String vertexName, int taskIndex) { } public static String getTaskAttemptIdentifier(int taskIndex, - int taskAttemptNumber) { + int taskAttemptNumber) { return String.format("%d_%d", taskIndex, taskAttemptNumber); } // TODO Maybe include a dag name in this. public static String getTaskAttemptIdentifier(String vertexName, - int taskIndex, int taskAttemptNumber) { + int taskIndex, int taskAttemptNumber) { return String.format("%s_%06d_%02d", vertexName, taskIndex, taskAttemptNumber); } @@ -84,29 +84,29 @@ public static Combiner instantiateCombiner(Configuration conf, TaskContext taskC } catch (ClassNotFoundException e) { throw new IOException("Unable to load combiner class: " + className); } - + Combiner combiner = null; - - Constructor ctor; - try { - ctor = clazz.getConstructor(TaskContext.class); - combiner = ctor.newInstance(taskContext); - } catch (SecurityException e) { - throw new IOException(e); - } catch (NoSuchMethodException e) { - throw new IOException(e); - } catch (IllegalArgumentException e) { - throw new IOException(e); - } catch (InstantiationException e) { - throw new IOException(e); - } catch (IllegalAccessException e) { - throw new IOException(e); - } catch (InvocationTargetException e) { - throw new IOException(e); - } - return combiner; + + Constructor ctor; + try { + ctor = clazz.getConstructor(TaskContext.class); + combiner = ctor.newInstance(taskContext); + } catch (SecurityException e) { + throw new IOException(e); + } catch (NoSuchMethodException e) { + throw new IOException(e); + } catch (IllegalArgumentException e) { + throw new IOException(e); + } catch (InstantiationException e) { + throw new IOException(e); + } catch (IllegalAccessException e) { + throw new IOException(e); + } catch (InvocationTargetException e) { + throw new IOException(e); + } + return combiner; } - + @SuppressWarnings("unchecked") public static Partitioner instantiatePartitioner(Configuration conf) throws IOException { @@ -166,7 +166,7 @@ public static TezTaskOutput instantiateTaskOutputManager(Configuration conf, Out throw new TezUncheckedException( "Unable to instantiate configured TezOutputFileManager: " + conf.get(Constants.TEZ_RUNTIME_TASK_OUTPUT_MANAGER, - TezTaskOutputFiles.class.getName()), e); + TezTaskOutputFiles.class.getName()), e); } } @@ -188,8 +188,8 @@ public static URL constructBaseURIForShuffleHandlerDagComplete( } public static URL constructBaseURIForShuffleHandlerVertexComplete( - String host, int port, String appId, int dagIdentifier, String vertexIndentifier, boolean sslShuffle) - throws MalformedURLException { + String host, int port, String appId, int dagIdentifier, String vertexIndentifier, boolean sslShuffle) + throws MalformedURLException { String httpProtocol = (sslShuffle) ? "https://" : "http://"; StringBuilder sb = new StringBuilder(httpProtocol); sb.append(host); diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ValuesIterator.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ValuesIterator.java index 4a75cbdd04..fe4f076eb0 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ValuesIterator.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ValuesIterator.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.runtime.library.common; @@ -37,14 +37,14 @@ /** * Iterates values while keys match in sorted input. - * + * * This class is not thread safe. Accessing methods from multiple threads will * lead to corrupt data. - * + * */ @Private -public class ValuesIterator { +public class ValuesIterator { protected TezRawKeyValueIterator in; //input iterator private KEY key; // current key private KEY nextKey; @@ -58,20 +58,20 @@ public class ValuesIterator { private DataInputBuffer valueIn = new DataInputBuffer(); private TezCounter inputKeyCounter; private TezCounter inputValueCounter; - + private int keyCtr = 0; private boolean hasMoreValues; // For the current key. private boolean isFirstRecord = true; private boolean completedProcessing; - - public ValuesIterator (TezRawKeyValueIterator in, - RawComparator comparator, - Class keyClass, - Class valClass, Configuration conf, - TezCounter inputKeyCounter, - TezCounter inputValueCounter) - throws IOException { + + public ValuesIterator(TezRawKeyValueIterator in, + RawComparator comparator, + Class keyClass, + Class valClass, Configuration conf, + TezCounter inputKeyCounter, + TezCounter inputValueCounter) + throws IOException { this.in = in; this.comparator = comparator; this.inputKeyCounter = inputKeyCounter; @@ -83,12 +83,12 @@ public ValuesIterator (TezRawKeyValueIterator in, this.valDeserializer.open(this.valueIn); } - TezRawKeyValueIterator getRawIterator() { return in; } + TezRawKeyValueIterator getRawIterator() {return in;} /** * Move to the next K-Vs pair * @return true if another pair exists, otherwise false. - * @throws IOException + * @throws IOException */ public boolean moveToNext() throws IOException { if (isFirstRecord) { @@ -107,22 +107,22 @@ public boolean moveToNext() throws IOException { } /** The current key. */ - public KEY getKey() { - return key; + public KEY getKey() { + return key; } - + // TODO NEWTEZ Maybe add another method which returns an iterator instead of iterable - + public Iterable getValues() { return new Iterable() { @Override public Iterator iterator() { - + return new Iterator() { private final int keyNumber = keyCtr; - + @Override public boolean hasNext() { return hasMoreValues; @@ -137,12 +137,12 @@ public VALUE next() { .checkState( keyNumber == keyCtr, "Cannot use values iterator on the previous K-V pair after moveToNext has been invoked to move to the next K-V pair"); - + try { readNextValue(); readNextKey(); } catch (IOException ie) { - throw new RuntimeException("problem advancing post rec#"+keyCtr, ie); + throw new RuntimeException("problem advancing post rec#" + keyCtr, ie); } inputValueCounter.increment(1); return value; @@ -156,13 +156,11 @@ public void remove() { } }; } - - /** Start processing next unique key. */ private void nextKey() throws IOException { // read until we find a new key - while (hasMoreValues) { + while (hasMoreValues) { readNextKey(); } @@ -173,12 +171,12 @@ private void nextKey() throws IOException { hasMoreValues = more; } - /** + /** * read the next key - which may be the same as the current key. */ private void readNextKey() throws IOException { more = in.next(); - if (more) { + if (more) { DataInputBuffer nextKeyBytes = in.getKey(); if (!in.isSameKey()) { keyIn.reset(nextKeyBytes.getData(), nextKeyBytes.getPosition(), @@ -189,7 +187,7 @@ private void readNextKey() throws IOException { if (key == null || false == hasMoreValues) { // invariant: more=true & there are no more values in an existing key group // so this indicates start of new key group - if(inputKeyCounter != null) { + if (inputKeyCounter != null) { inputKeyCounter.increment(1); } ++keyCtr; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/combine/Combiner.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/combine/Combiner.java index 43d2caa0f8..88e8da78b5 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/combine/Combiner.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/combine/Combiner.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -29,8 +29,8 @@ /** *Combiner Initialization

    The Combiner class is picked up * using the TEZ_RUNTIME_COMBINER_CLASS attribute in {@link TezRuntimeConfiguration} - * - * + * + * * Partitioners need to provide a single argument ({@link TezRawKeyValueIterator}) * constructor. */ diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/comparator/ProxyComparator.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/comparator/ProxyComparator.java index 5297f7cbd5..39db96e259 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/comparator/ProxyComparator.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/comparator/ProxyComparator.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -42,5 +42,4 @@ public interface ProxyComparator extends RawComparator { * @return proxy */ int getProxy(KEY key); - } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/comparator/TezBytesComparator.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/comparator/TezBytesComparator.java index 81df0cf7db..5cece13ba5 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/comparator/TezBytesComparator.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/comparator/TezBytesComparator.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,7 +36,7 @@ public TezBytesComparator() { */ @Override public int compare(byte[] b1, int s1, int l1, - byte[] b2, int s2, int l2) { + byte[] b2, int s2, int l2) { return compareBytes(b1, s1, l1, b2, s2, l2); } @@ -47,17 +47,16 @@ public int getProxy(BytesWritable key) { final byte[] content = key.getBytes(); int b1 = 0, b2 = 0, b3 = 0; switch (len) { - default: - case 3: - b3 = content[2] & 0xff; - case 2: - b2 = content[1] & 0xff; - case 1: - b1 = content[0] & 0xff; - case 0: + default: + case 3: + b3 = content[2] & 0xff; + case 2: + b2 = content[1] & 0xff; + case 1: + b1 = content[0] & 0xff; + case 0: } prefix = (b1 << 16) | (b2 << 8) | (b3); return prefix; } - } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/readers/UnorderedKVReader.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/readers/UnorderedKVReader.java index 60f70ac271..311e948035 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/readers/UnorderedKVReader.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/readers/UnorderedKVReader.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -47,10 +47,10 @@ public class UnorderedKVReader extends KeyValueReader { private static final Logger LOG = LoggerFactory.getLogger(UnorderedKVReader.class); - + private final ShuffleManager shuffleManager; private final CompressionCodec codec; - + private final Class keyClass; private final Class valClass; private final Deserializer keyDeserializer; @@ -61,26 +61,25 @@ public class UnorderedKVReader extends KeyValueReader { private final boolean ifileReadAhead; private final int ifileReadAheadLength; private final int ifileBufferSize; - + private final TezCounter inputRecordCounter; private final InputContext context; - + private K key; private V value; - + private FetchedInput currentFetchedInput; private IFile.Reader currentReader; - + // TODO Remove this once per I/O counters are separated properly. Relying on // the counter at the moment will generate aggregate numbers. private int numRecordsRead = 0; private final AtomicLong totalBytesRead = new AtomicLong(0); private final AtomicLong totalFileBytes = new AtomicLong(0); - public UnorderedKVReader(ShuffleManager shuffleManager, Configuration conf, - CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLength, int ifileBufferSize, - TezCounter inputRecordCounter, InputContext context) + CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLength, int ifileBufferSize, + TezCounter inputRecordCounter, InputContext context) throws IOException { this.shuffleManager = shuffleManager; this.context = context; @@ -105,16 +104,16 @@ public UnorderedKVReader(ShuffleManager shuffleManager, Configuration conf, } // TODO NEWTEZ Maybe add an interface to check whether next will block. - + /** * Moves to the next key/values(s) pair - * + * * @return true if another key/value(s) pair exists, false if there are no * more. * @throws IOException * if an error occurs */ - @Override + @Override public boolean next() throws IOException { if (readNextFromCurrentReader()) { inputRecordCounter.increment(1); @@ -124,7 +123,7 @@ public boolean next() throws IOException { } else { boolean nextInputExists = moveToNextInput(); while (nextInputExists) { - if(readNextFromCurrentReader()) { + if (readNextFromCurrentReader()) { inputRecordCounter.increment(1); context.notifyProgress(); numRecordsRead++; @@ -138,7 +137,6 @@ public boolean next() throws IOException { } } - @Override public Object getCurrentKey() throws IOException { return (Object) key; @@ -153,12 +151,13 @@ public float getProgress() throws IOException, InterruptedException { final int numInputs = shuffleManager.getNumInputs(); if (totalFileBytes.get() > 0 && numInputs > 0) { return ((1.0f) * (totalBytesRead.get() + ((currentReader != null) ? currentReader.bytesRead : - 0.0f)) / + 0.0f)) / totalFileBytes.get()) * (shuffleManager.getNumCompletedInputsFloat() / (1.0f * numInputs)); } return 0.0f; } + /** * Tries reading the next key and value from the current reader. * @return true if the current reader has more records @@ -179,11 +178,11 @@ private boolean readNextFromCurrentReader() throws IOException { return false; } } - + /** * Moves to the next available input. This method may block if the input is not ready yet. * Also takes care of closing the previous input. - * + * * @return true if the next input exists, false otherwise * @throws IOException */ diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/security/SecureShuffleUtils.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/security/SecureShuffleUtils.java index 67b8de27c5..2d1b3a0277 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/security/SecureShuffleUtils.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/security/SecureShuffleUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,7 +17,6 @@ */ package org.apache.tez.runtime.library.common.security; - import java.io.IOException; import java.net.URL; @@ -31,7 +30,7 @@ import org.apache.tez.common.security.JobTokenSecretManager; /** - * + * * utilities for generating kyes, hashes and verifying them for shuffle * */ @@ -40,7 +39,7 @@ public class SecureShuffleUtils { public static final String HTTP_HEADER_URL_HASH = "UrlHash"; public static final String HTTP_HEADER_REPLY_URL_HASH = "ReplyHash"; - + /** * Base64 encoded hash of msg * @param msg @@ -112,7 +111,7 @@ public static void verifyReply(String base64Hash, String msg, SecretKey key) thr byte[] hash = Base64.decodeBase64(base64Hash.getBytes(Charsets.UTF_8)); boolean res = verifyHash(hash, msg.getBytes(Charsets.UTF_8), key); - if(res != true) { + if (res != true) { throw new IOException("Verification of the hashReply failed"); } } @@ -129,11 +128,11 @@ public static void verifyReply(String base64Hash, String msg, JobTokenSecretMana boolean res = verifyHash(hash, msg.getBytes(Charsets.UTF_8), mgr); - if(res != true) { + if (res != true) { throw new IOException("Verification of the hashReply failed"); } } - + /** * Shuffle specific utils - build string for encoding from URL * @param url diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/serializer/SerializationContext.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/serializer/SerializationContext.java index 2398b8f930..f20c02645c 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/serializer/SerializationContext.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/serializer/SerializationContext.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -47,7 +47,7 @@ public SerializationContext(Configuration conf) { } public SerializationContext(Class keyClass, Class valueClass, - Serialization keySerialization, Serialization valSerialization) { + Serialization keySerialization, Serialization valSerialization) { this.keyClass = keyClass; this.valueClass = valueClass; this.keySerialization = keySerialization; @@ -70,12 +70,12 @@ public Serialization getValSerialization() { return valSerialization; } - @SuppressWarnings({ "rawtypes", "unchecked" }) + @SuppressWarnings({"rawtypes", "unchecked"}) public Serializer getKeySerializer() { return keySerialization.getSerializer((Class) keyClass); } - @SuppressWarnings({ "rawtypes", "unchecked" }) + @SuppressWarnings({"rawtypes", "unchecked"}) public Serializer getValueSerializer() { return valSerialization.getSerializer((Class) valueClass); } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/serializer/TezBytesWritableSerialization.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/serializer/TezBytesWritableSerialization.java index 7716c5951c..b5d6c2ebff 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/serializer/TezBytesWritableSerialization.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/serializer/TezBytesWritableSerialization.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -104,7 +104,6 @@ public Writable deserialize(Writable w) throws IOException { public void close() throws IOException { dataIn.close(); } - } public static class TezBytesWritableSerializer extends Configured implements @@ -129,4 +128,3 @@ public void close() throws IOException { } } } - diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/DiskFetchedInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/DiskFetchedInput.java index 22b2899e1e..70fd23f358 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/DiskFetchedInput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/DiskFetchedInput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,16 +36,16 @@ public class DiskFetchedInput extends FetchedInput { private static final Logger LOG = LoggerFactory.getLogger(DiskFetchedInput.class); - + private final FileSystem localFS; private final Path tmpOutputPath; private final Path outputPath; private final long size; public DiskFetchedInput(long compressedSize, - InputAttemptIdentifier inputAttemptIdentifier, - FetchedInputCallback callbackHandler, Configuration conf, - LocalDirAllocator localDirAllocator, TezTaskOutputFiles filenameAllocator) + InputAttemptIdentifier inputAttemptIdentifier, + FetchedInputCallback callbackHandler, Configuration conf, + LocalDirAllocator localDirAllocator, TezTaskOutputFiles filenameAllocator) throws IOException { super(inputAttemptIdentifier, callbackHandler); @@ -85,7 +85,7 @@ public final Path getInputPath() { } return this.tmpOutputPath; } - + @Override public void commit() throws IOException { if (isState(State.PENDING)) { @@ -104,7 +104,7 @@ public void abort() throws IOException { notifyFetchFailure(); } } - + @Override public void free() { Preconditions.checkState( diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchResult.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchResult.java index d8b8998655..740baa1825 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchResult.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchResult.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -30,12 +30,12 @@ public class FetchResult { private final String additionalInfo; public FetchResult(String host, int port, int partition, int partitionCount, - Iterable pendingInputs) { + Iterable pendingInputs) { this(host, port, partition, partitionCount, pendingInputs, null); } public FetchResult(String host, int port, int partition, int partitionCount, - Iterable pendingInputs, String additionalInfo) { + Iterable pendingInputs, String additionalInfo) { this.host = host; this.port = port; this.partition = partition; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchedInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchedInput.java index 8982c27cad..b31c2472bf 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchedInput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchedInput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -28,14 +28,14 @@ @Private public abstract class FetchedInput { - + public static enum Type { WAIT, // TODO NEWTEZ Implement this, only if required. MEMORY, DISK, DISK_DIRECT } - + protected static enum State { PENDING, COMMITTED, ABORTED, FREED } @@ -48,7 +48,7 @@ protected static enum State { private byte state; protected FetchedInput(InputAttemptIdentifier inputAttemptIdentifier, - FetchedInputCallback callbackHandler) { + FetchedInputCallback callbackHandler) { this.inputAttemptIdentifier = inputAttemptIdentifier; this.callback = callbackHandler; this.id = ID_GEN.getAndIncrement(); @@ -99,7 +99,7 @@ public InputAttemptIdentifier getInputAttemptIdentifier() { public void notifyFetchComplete() { this.callback.fetchComplete(this); } - + /** * Inform the Allocator about a failed resource. * This should be called by abort @@ -107,7 +107,7 @@ public void notifyFetchComplete() { public void notifyFetchFailure() { this.callback.fetchFailed(this); } - + /** * Inform the Allocator about a completed resource being released. * This should be called by free @@ -115,7 +115,7 @@ public void notifyFetchFailure() { public void notifyFreedResource() { this.callback.freeResources(this); } - + /** * Returns the output stream to be used to write fetched data. Users are * expected to close the OutputStream when they're done @@ -144,7 +144,7 @@ public void notifyFreedResource() { * reclaimed. */ public abstract void free(); - + @Override public int hashCode() { return id; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchedInputAllocator.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchedInputAllocator.java index 74fa26a454..9d06baf4cc 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchedInputAllocator.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchedInputAllocator.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,9 +26,8 @@ public interface FetchedInputAllocator { public FetchedInput allocate(long actualSize, long compresedSize, - InputAttemptIdentifier inputAttemptIdentifier) throws IOException; - + InputAttemptIdentifier inputAttemptIdentifier) throws IOException; + public FetchedInput allocateType(Type type, long actualSize, long compresedSize, - InputAttemptIdentifier inputAttemptIdentifier) throws IOException; - + InputAttemptIdentifier inputAttemptIdentifier) throws IOException; } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchedInputCallback.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchedInputCallback.java index a89604169d..c80ae274f0 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchedInputCallback.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetchedInputCallback.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,11 +19,10 @@ package org.apache.tez.runtime.library.common.shuffle; public interface FetchedInputCallback { - + public void fetchComplete(FetchedInput fetchedInput); - + public void fetchFailed(FetchedInput fetchedInput); - + public void freeResources(FetchedInput fetchedInput); - } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java index eb34ec2993..d30534bb4e 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/Fetcher.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -132,24 +132,25 @@ public String toString() { private boolean ifileReadAhead = TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_DEFAULT; private int ifileReadAheadLength = TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES_DEFAULT; - + private final JobTokenSecretManager jobTokenSecretMgr; private final FetcherCallback fetcherCallback; private final FetchedInputAllocator inputManager; private final ApplicationId appId; private final int dagIdentifier; - + private final String logIdentifier; private final String localHostname; - + private final AtomicBoolean isShutDown = new AtomicBoolean(false); protected final int fetcherIdentifier; // Parameters to track work. private List srcAttempts; + @VisibleForTesting public List getSrcAttempts() { return srcAttempts; @@ -159,6 +160,7 @@ public List getSrcAttempts() { Map srcAttemptsRemaining; private String host; + @VisibleForTesting public String getHost() { return host; @@ -173,7 +175,7 @@ public String getHost() { private URL url; private volatile DataInputStream input; - + BaseHttpConnection httpConnection; private HttpConnectionParams httpConnectionParams; @@ -195,15 +197,15 @@ public String getHost() { private final boolean isDebugEnabled = LOG.isDebugEnabled(); protected Fetcher(FetcherCallback fetcherCallback, HttpConnectionParams params, - FetchedInputAllocator inputManager, InputContext inputContext, - JobTokenSecretManager jobTokenSecretManager, Configuration conf, - RawLocalFileSystem localFs, - LocalDirAllocator localDirAllocator, - Path lockPath, - boolean localDiskFetchEnabled, - boolean sharedFetchEnabled, - String localHostname, - int shufflePort, boolean asyncHttp, boolean verifyDiskChecksum, boolean compositeFetch) { + FetchedInputAllocator inputManager, InputContext inputContext, + JobTokenSecretManager jobTokenSecretManager, Configuration conf, + RawLocalFileSystem localFs, + LocalDirAllocator localDirAllocator, + Path lockPath, + boolean localDiskFetchEnabled, + boolean sharedFetchEnabled, + String localHostname, + int shufflePort, boolean asyncHttp, boolean verifyDiskChecksum, boolean compositeFetch) { this.asyncHttp = asyncHttp; this.verifyDiskChecksum = verifyDiskChecksum; this.fetcherCallback = fetcherCallback; @@ -222,7 +224,7 @@ protected Fetcher(FetcherCallback fetcherCallback, HttpConnectionParams params, String sourceDestNameTrimmed = TezUtilsInternal.cleanVertexName(inputContext.getSourceVertexName()) + " -> " + TezUtilsInternal.cleanVertexName(inputContext.getTaskVertexName()); - this.logIdentifier = " fetcher [" + sourceDestNameTrimmed +"] " + fetcherIdentifier; + this.logIdentifier = " fetcher [" + sourceDestNameTrimmed + "] " + fetcherIdentifier; this.localFs = localFs; this.localDirAllocator = localDirAllocator; @@ -240,7 +242,6 @@ protected Fetcher(FetcherCallback fetcherCallback, HttpConnectionParams params, } } - // helper method to populate the remaining map void populateRemainingMap(List origlist) { if (srcAttemptsRemaining == null) { @@ -262,7 +263,7 @@ public FetchResult callInternal() throws Exception { populateRemainingMap(srcAttempts); for (InputAttemptIdentifier in : srcAttemptsRemaining.values()) { if (in instanceof CompositeInputAttemptIdentifier) { - CompositeInputAttemptIdentifier cin = (CompositeInputAttemptIdentifier)in; + CompositeInputAttemptIdentifier cin = (CompositeInputAttemptIdentifier) in; for (int i = 0; i < cin.getInputIdentifierCount(); i++) { pathToAttemptMap.put(new PathPartition(cin.getPathComponent(), partition + i), cin.expand(i)); } @@ -287,7 +288,7 @@ public FetchResult callInternal() throws Exception { hostFetchResult = setupLocalDiskFetch(); } else if (multiplex) { hostFetchResult = doSharedFetch(); - } else{ + } else { hostFetchResult = doHttpFetch(); } @@ -321,8 +322,8 @@ public FetchResult callInternal() throws Exception { private final class CachingCallBack { // this is a closure object wrapping this in an inner class public void cache(String host, - InputAttemptIdentifier srcAttemptId, FetchedInput fetchedInput, - long compressedLength, long decompressedLength) { + InputAttemptIdentifier srcAttemptId, FetchedInput fetchedInput, + long compressedLength, long decompressedLength) { try { // this breaks badly on partitioned input - please use responsibly Preconditions.checkArgument(partition == 0, "Partition == 0"); @@ -331,7 +332,7 @@ public void cache(String host, final Path outputPath = localDirAllocator.getLocalPathForWrite(finalOutput, compressedLength, conf); final TezSpillRecord spillRec = new TezSpillRecord(1); final TezIndexRecord indexRec; - Path tmpIndex = outputPath.suffix(Constants.TEZ_RUNTIME_TASK_OUTPUT_INDEX_SUFFIX_STRING+tmpSuffix); + Path tmpIndex = outputPath.suffix(Constants.TEZ_RUNTIME_TASK_OUTPUT_INDEX_SUFFIX_STRING + tmpSuffix); if (localFs.exists(tmpIndex)) { LOG.warn("Found duplicate instance of input index file " + tmpIndex); @@ -341,26 +342,26 @@ public void cache(String host, Path tmpPath = null; switch (fetchedInput.getType()) { - case DISK: { - DiskFetchedInput input = (DiskFetchedInput) fetchedInput; - indexRec = new TezIndexRecord(0, decompressedLength, compressedLength); - localFs.mkdirs(outputPath.getParent()); - // avoid pit-falls of speculation - tmpPath = outputPath.suffix(tmpSuffix); - // JDK7 - TODO: use Files implementation to speed up this process - localFs.copyFromLocalFile(input.getInputPath(), tmpPath); - // rename is atomic - boolean renamed = localFs.rename(tmpPath, outputPath); - if(!renamed) { - LOG.warn("Could not rename to cached file name " + outputPath); - localFs.delete(tmpPath, false); - return; + case DISK: { + DiskFetchedInput input = (DiskFetchedInput) fetchedInput; + indexRec = new TezIndexRecord(0, decompressedLength, compressedLength); + localFs.mkdirs(outputPath.getParent()); + // avoid pit-falls of speculation + tmpPath = outputPath.suffix(tmpSuffix); + // JDK7 - TODO: use Files implementation to speed up this process + localFs.copyFromLocalFile(input.getInputPath(), tmpPath); + // rename is atomic + boolean renamed = localFs.rename(tmpPath, outputPath); + if (!renamed) { + LOG.warn("Could not rename to cached file name " + outputPath); + localFs.delete(tmpPath, false); + return; + } } - } - break; - default: - LOG.warn("Incorrect use of CachingCallback for " + srcAttemptId); - return; + break; + default: + LOG.warn("Incorrect use of CachingCallback for " + srcAttemptId); + return; } spillRec.putIndex(indexRec, 0); @@ -374,7 +375,7 @@ public void cache(String host, localFs.delete(outputPath, false); LOG.warn("Could not rename the index file to " + outputPath - .suffix(Constants.TEZ_RUNTIME_TASK_OUTPUT_INDEX_SUFFIX_STRING)); + .suffix(Constants.TEZ_RUNTIME_TASK_OUTPUT_INDEX_SUFFIX_STRING)); return; } } catch (IOException ioe) { @@ -551,7 +552,7 @@ private HostFetchResult setupConnection(Collection attem "Fetch Failure while connecting from %s to: %s:%d, attempt: %s Informing ShuffleManager: ", localHostname, host, port, firstAttempt), e); return new HostFetchResult(new FetchResult(host, port, partition, partitionCount, srcAttemptsRemaining.values()), - new InputAttemptFetchFailure[] { new InputAttemptFetchFailure(firstAttempt) }, true); + new InputAttemptFetchFailure[]{new InputAttemptFetchFailure(firstAttempt)}, true); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); //reset status @@ -560,7 +561,6 @@ private HostFetchResult setupConnection(Collection attem return null; } - protected void setupConnectionInternal(String host, Collection attempts) throws IOException, InterruptedException { input = httpConnection.getInputStream(); @@ -717,7 +717,7 @@ public void freeResources(FetchedInput fetchedInput) { } } } - if(!hasFailures) { + if (!hasFailures) { iterator.remove(); } } @@ -758,7 +758,7 @@ private final String getMapOutputFile(String pathComponent) { pathComponent + Path.SEPARATOR + Constants.TEZ_RUNTIME_TASK_OUTPUT_FILENAME_STRING; - if(ShuffleUtils.isTezShuffleHandler(conf)) { + if (ShuffleUtils.isTezShuffleHandler(conf)) { return Constants.DAG_PREFIX + this.dagIdentifier + Path.SEPARATOR + outputPath; } @@ -845,7 +845,7 @@ public String toString() { @VisibleForTesting InputAttemptFetchFailure[] fetchInputs(DataInputStream input, CachingCallBack callback, - InputAttemptIdentifier inputAttemptIdentifier) + InputAttemptIdentifier inputAttemptIdentifier) throws FetcherReadTimeoutException { FetchedInput fetchedInput = null; InputAttemptIdentifier srcAttemptId = null; @@ -875,8 +875,8 @@ InputAttemptFetchFailure[] fetchInputs(DataInputStream input, CachingCallBack ca + InputAttemptIdentifier.PATH_PREFIX + ", partition: " + header.getPartition() + " while fetching " + inputAttemptIdentifier); // this should be treated as local fetch failure while reporting later - return new InputAttemptFetchFailure[] { - InputAttemptFetchFailure.fromDiskErrorAtSource(inputAttemptIdentifier) }; + return new InputAttemptFetchFailure[]{ + InputAttemptFetchFailure.fromDiskErrorAtSource(inputAttemptIdentifier)}; } else { throw new IllegalArgumentException( "Invalid map id: " + header.getMapId() + ", expected to start with " @@ -926,8 +926,8 @@ InputAttemptFetchFailure[] fetchInputs(DataInputStream input, CachingCallBack ca srcAttemptId = getNextRemainingAttempt(); } assert (srcAttemptId != null); - return new InputAttemptFetchFailure[] { - InputAttemptFetchFailure.fromAttempt(srcAttemptId) }; + return new InputAttemptFetchFailure[]{ + InputAttemptFetchFailure.fromAttempt(srcAttemptId)}; } else { if (isDebugEnabled) { LOG.debug("Already shutdown. Ignoring verification failure."); @@ -1031,8 +1031,8 @@ InputAttemptFetchFailure[] fetchInputs(DataInputStream input, CachingCallBack ca if (srcAttemptId == null) { return InputAttemptFetchFailure.fromAttempts(srcAttemptsRemaining.values()); } else { - return new InputAttemptFetchFailure[] { - new InputAttemptFetchFailure(srcAttemptId) }; + return new InputAttemptFetchFailure[]{ + new InputAttemptFetchFailure(srcAttemptId)}; } } LOG.warn("Failed to shuffle output of " + srcAttemptId + " from " + host + " (to " @@ -1041,8 +1041,8 @@ InputAttemptFetchFailure[] fetchInputs(DataInputStream input, CachingCallBack ca // Cleanup the fetchedInput cleanupFetchedInput(fetchedInput); // metrics.failedFetch(); - return new InputAttemptFetchFailure[] { - new InputAttemptFetchFailure(srcAttemptId) }; + return new InputAttemptFetchFailure[]{ + new InputAttemptFetchFailure(srcAttemptId)}; } return null; } @@ -1077,7 +1077,7 @@ private boolean shouldRetry(InputAttemptIdentifier srcAttemptId, Throwable ioe) if (currentTime - retryStartTime < httpConnectionParams.getReadTimeout()) { LOG.warn("Shuffle output from " + srcAttemptId + - " failed (to "+ localHostname +"), retry it."); + " failed (to " + localHostname + "), retry it."); //retry connecting to the host return true; } else { @@ -1090,7 +1090,7 @@ private boolean shouldRetry(InputAttemptIdentifier srcAttemptId, Throwable ioe) /** * Do some basic verification on the input received -- Being defensive - * + * * @param compressedLength * @param decompressedLength * @param fetchPartition @@ -1099,7 +1099,7 @@ private boolean shouldRetry(InputAttemptIdentifier srcAttemptId, Throwable ioe) * @return true/false, based on if the verification succeeded or not */ private boolean verifySanity(long compressedLength, long decompressedLength, - int fetchPartition, InputAttemptIdentifier srcAttemptId, String pathComponent) { + int fetchPartition, InputAttemptIdentifier srcAttemptId, String pathComponent) { if (compressedLength < 0 || decompressedLength < 0) { // wrongLengthErrs.increment(1); LOG.warn(" invalid lengths in input header -> headerPathComponent: " @@ -1120,7 +1120,7 @@ private boolean verifySanity(long compressedLength, long decompressedLength, } return true; } - + private InputAttemptIdentifier getNextRemainingAttempt() { if (srcAttemptsRemaining.size() > 0) { return srcAttemptsRemaining.values().iterator().next(); @@ -1137,21 +1137,21 @@ public static class FetcherBuilder { private boolean workAssigned = false; public FetcherBuilder(FetcherCallback fetcherCallback, - HttpConnectionParams params, FetchedInputAllocator inputManager, InputContext inputContext, - JobTokenSecretManager jobTokenSecretMgr, Configuration conf, boolean localDiskFetchEnabled, - String localHostname, int shufflePort, boolean asyncHttp, boolean verifyDiskChecksum, boolean compositeFetch) { + HttpConnectionParams params, FetchedInputAllocator inputManager, InputContext inputContext, + JobTokenSecretManager jobTokenSecretMgr, Configuration conf, boolean localDiskFetchEnabled, + String localHostname, int shufflePort, boolean asyncHttp, boolean verifyDiskChecksum, boolean compositeFetch) { this.fetcher = new Fetcher(fetcherCallback, params, inputManager, inputContext, jobTokenSecretMgr, conf, null, null, null, localDiskFetchEnabled, false, localHostname, shufflePort, asyncHttp, verifyDiskChecksum, compositeFetch); } public FetcherBuilder(FetcherCallback fetcherCallback, - HttpConnectionParams params, FetchedInputAllocator inputManager, InputContext inputContext, - JobTokenSecretManager jobTokenSecretMgr, Configuration conf, RawLocalFileSystem localFs, - LocalDirAllocator localDirAllocator, Path lockPath, - boolean localDiskFetchEnabled, boolean sharedFetchEnabled, - String localHostname, int shufflePort, boolean asyncHttp, boolean verifyDiskChecksum, boolean compositeFetch, - boolean enableFetcherTestingErrors) { + HttpConnectionParams params, FetchedInputAllocator inputManager, InputContext inputContext, + JobTokenSecretManager jobTokenSecretMgr, Configuration conf, RawLocalFileSystem localFs, + LocalDirAllocator localDirAllocator, Path lockPath, + boolean localDiskFetchEnabled, boolean sharedFetchEnabled, + String localHostname, int shufflePort, boolean asyncHttp, boolean verifyDiskChecksum, boolean compositeFetch, + boolean enableFetcherTestingErrors) { if (enableFetcherTestingErrors) { this.fetcher = new FetcherWithInjectableErrors(fetcherCallback, params, inputManager, inputContext, jobTokenSecretMgr, conf, localFs, localDirAllocator, @@ -1182,7 +1182,7 @@ public FetcherBuilder setIFileParams(boolean readAhead, int readAheadBytes) { } public FetcherBuilder assignWork(String host, int port, int partition, int partitionCount, - List inputs) { + List inputs) { fetcher.host = host; fetcher.port = port; fetcher.partition = partition; @@ -1218,4 +1218,3 @@ public boolean equals(Object obj) { return true; } } - diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetcherCallback.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetcherCallback.java index b751fb9ce0..96b91c548b 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetcherCallback.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetcherCallback.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,10 +25,9 @@ public interface FetcherCallback { public void fetchSucceeded(String host, InputAttemptIdentifier srcAttemptIdentifier, - FetchedInput fetchedInput, long fetchedBytes, long decompressedLength, long copyDuration) + FetchedInput fetchedInput, long fetchedBytes, long decompressedLength, long copyDuration) throws IOException; - - public void fetchFailed(String host, InputAttemptFetchFailure srcAttemptFetchFailure, - boolean connectFailed); + public void fetchFailed(String host, InputAttemptFetchFailure srcAttemptFetchFailure, + boolean connectFailed); } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetcherErrorTestingConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetcherErrorTestingConfig.java index ce15a87fb9..3e9a4e9cbd 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetcherErrorTestingConfig.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetcherErrorTestingConfig.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetcherWithInjectableErrors.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetcherWithInjectableErrors.java index cf53a57a9e..f8dbab542a 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetcherWithInjectableErrors.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/FetcherWithInjectableErrors.java @@ -35,11 +35,11 @@ public class FetcherWithInjectableErrors extends Fetcher { private String srcNameTrimmed; protected FetcherWithInjectableErrors(FetcherCallback fetcherCallback, HttpConnectionParams params, - FetchedInputAllocator inputManager, InputContext inputContext, - JobTokenSecretManager jobTokenSecretManager, Configuration conf, - RawLocalFileSystem localFs, LocalDirAllocator localDirAllocator, Path lockPath, boolean localDiskFetchEnabled, - boolean sharedFetchEnabled, String localHostname, int shufflePort, boolean asyncHttp, boolean verifyDiskChecksum, - boolean compositeFetch) { + FetchedInputAllocator inputManager, InputContext inputContext, + JobTokenSecretManager jobTokenSecretManager, Configuration conf, + RawLocalFileSystem localFs, LocalDirAllocator localDirAllocator, Path lockPath, boolean localDiskFetchEnabled, + boolean sharedFetchEnabled, String localHostname, int shufflePort, boolean asyncHttp, boolean verifyDiskChecksum, + boolean compositeFetch) { super(fetcherCallback, params, inputManager, inputContext, jobTokenSecretManager, conf, localFs, localDirAllocator, lockPath, localDiskFetchEnabled, sharedFetchEnabled, localHostname, shufflePort, asyncHttp, verifyDiskChecksum, compositeFetch); diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/HostPort.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/HostPort.java index cac9d9a8c3..2410cf5600 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/HostPort.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/HostPort.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputAttemptFetchFailure.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputAttemptFetchFailure.java index 4ce1699cf5..0de0b4f7b5 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputAttemptFetchFailure.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputAttemptFetchFailure.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -40,7 +40,7 @@ public InputAttemptFetchFailure(InputAttemptIdentifier inputAttemptIdentifier) { } public InputAttemptFetchFailure(InputAttemptIdentifier inputAttemptIdentifier, - boolean isLocalFetch, boolean isDiskErrorAtSource) { + boolean isLocalFetch, boolean isDiskErrorAtSource) { this.inputAttemptIdentifier = inputAttemptIdentifier; this.isLocalFetch = isLocalFetch; this.isDiskErrorAtSource = isDiskErrorAtSource; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputHost.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputHost.java index 6014b84e80..80d19c21c6 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputHost.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/InputHost.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -30,7 +30,7 @@ /** * Represents a Host with respect to the MapReduce ShuffleHandler. - * + * */ public class InputHost extends HostPort { @@ -95,7 +95,7 @@ public int getNumPendingPartitions() { } public synchronized void addKnownInput(int partition, int partitionCount, - InputAttemptIdentifier srcAttempt) { + InputAttemptIdentifier srcAttempt) { PartitionRange partitionRange = new PartitionRange(partition, partitionCount); BlockingQueue inputs = partitionToInputs.get(partitionRange); @@ -123,7 +123,7 @@ public String toDetailedString() { return "HostPort=" + super.toString() + ", InputDetails=" + partitionToInputs; } - + @Override public String toString() { return "HostPort=" + super.toString() + ", PartitionIds=" + diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/LocalDiskFetchedInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/LocalDiskFetchedInput.java index d7ec265048..b115ff64ab 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/LocalDiskFetchedInput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/LocalDiskFetchedInput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -64,7 +64,7 @@ public long getSize() { @Override public OutputStream getOutputStream() throws IOException { - throw new IOException("Output Stream is not supported for " + this.toString()); + throw new IOException("Output Stream is not supported for " + this.toString()); } @Override @@ -89,7 +89,7 @@ public void abort() { notifyFetchFailure(); } } - + @Override public void free() { Preconditions.checkState( @@ -126,5 +126,4 @@ protected long getStartOffset() { protected FileSystem getLocalFS() { return localFS; } - } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/MemoryFetchedInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/MemoryFetchedInput.java index 63aefa8d13..f82eee8131 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/MemoryFetchedInput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/MemoryFetchedInput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -32,8 +32,8 @@ public class MemoryFetchedInput extends FetchedInput { private byte[] byteArray; public MemoryFetchedInput(long actualSize, - InputAttemptIdentifier inputAttemptIdentifier, - FetchedInputCallback callbackHandler) { + InputAttemptIdentifier inputAttemptIdentifier, + FetchedInputCallback callbackHandler) { super(inputAttemptIdentifier, callbackHandler); this.byteArray = new byte[(int) actualSize]; } @@ -64,7 +64,7 @@ public InputStream getInputStream() { public byte[] getBytes() { return byteArray; } - + @Override public void commit() { if (isState(State.PENDING)) { @@ -80,7 +80,7 @@ public void abort() { notifyFetchFailure(); } } - + @Override public void free() { Preconditions.checkState( diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleEventHandler.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleEventHandler.java index da7c9445ee..9091e83cd5 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleEventHandler.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleEventHandler.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,5 +25,6 @@ public interface ShuffleEventHandler { public void handleEvents(List events) throws IOException; + public void logProgress(boolean updateOnClose); } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java index ca8b84724e..3262003539 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -115,9 +115,9 @@ public static int deserializeShuffleProviderMetaData(ByteBuffer meta) } public static void shuffleToMemory(byte[] shuffleData, - InputStream input, int decompressedLength, int compressedLength, - CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLength, - Logger LOG, InputAttemptIdentifier identifier) throws IOException { + InputStream input, int decompressedLength, int compressedLength, + CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLength, + Logger LOG, InputAttemptIdentifier identifier) throws IOException { try { IFile.Reader.readToMemory(shuffleData, input, compressedLength, codec, ifileReadAhead, ifileReadAheadLength); @@ -141,10 +141,10 @@ public static void shuffleToMemory(byte[] shuffleData, } } } - + public static void shuffleToDisk(OutputStream output, String hostIdentifier, - InputStream input, long compressedLength, long decompressedLength, Logger LOG, InputAttemptIdentifier identifier, - boolean ifileReadAhead, int ifileReadAheadLength, boolean verifyChecksum) throws IOException { + InputStream input, long compressedLength, long decompressedLength, Logger LOG, InputAttemptIdentifier identifier, + boolean ifileReadAhead, int ifileReadAheadLength, boolean verifyChecksum) throws IOException { // Copy data to local-disk long bytesLeft = compressedLength; try { @@ -185,8 +185,8 @@ public static void shuffleToDisk(OutputStream output, String hostIdentifier, if (bytesLeft != 0) { throw new IOException("Incomplete map output received for " + identifier + " from " + - hostIdentifier + " (" + - bytesLeft + " bytes missing of " + + hostIdentifier + " (" + + bytesLeft + " bytes missing of " + compressedLength + ")"); } } @@ -204,7 +204,7 @@ public static void ioCleanup(Closeable... closeables) { } public static StringBuilder constructBaseURIForShuffleHandler(String host, - int port, int partition, int partitionCount, String appId, int dagIdentifier, boolean sslShuffle) { + int port, int partition, int partitionCount, String appId, int dagIdentifier, boolean sslShuffle) { final String http_protocol = (sslShuffle) ? "https://" : "http://"; StringBuilder sb = new StringBuilder(http_protocol); sb.append(host); @@ -226,7 +226,7 @@ public static StringBuilder constructBaseURIForShuffleHandler(String host, } public static URL constructInputURL(String baseURI, - Collection inputs, boolean keepAlive) throws MalformedURLException { + Collection inputs, boolean keepAlive) throws MalformedURLException { StringBuilder url = new StringBuilder(baseURI); boolean first = true; for (InputAttemptIdentifier input : inputs) { @@ -246,7 +246,7 @@ public static URL constructInputURL(String baseURI, } public static BaseHttpConnection getHttpConnection(boolean asyncHttp, URL url, - HttpConnectionParams params, String logIdentifier, JobTokenSecretManager jobTokenSecretManager) + HttpConnectionParams params, String logIdentifier, JobTokenSecretManager jobTokenSecretManager) throws IOException { return TezRuntimeUtils.getHttpConnection(asyncHttp, url, params, logIdentifier, jobTokenSecretManager); } @@ -283,8 +283,8 @@ public static String stringify(DataMovementEventPayloadProto dmProto) { * @throws IOException */ static ByteBuffer generateDMEPayload(boolean sendEmptyPartitionDetails, - int numPhysicalOutputs, TezSpillRecord spillRecord, OutputContext context, - int spillId, boolean finalMergeEnabled, boolean isLastEvent, String pathComponent, String auxiliaryService, Deflater deflater) + int numPhysicalOutputs, TezSpillRecord spillRecord, OutputContext context, + int spillId, boolean finalMergeEnabled, boolean isLastEvent, String pathComponent, String auxiliaryService, Deflater deflater) throws IOException { DataMovementEventPayloadProto.Builder payloadBuilder = DataMovementEventPayloadProto .newBuilder(); @@ -292,7 +292,7 @@ static ByteBuffer generateDMEPayload(boolean sendEmptyPartitionDetails, boolean outputGenerated = true; if (sendEmptyPartitionDetails) { BitSet emptyPartitionDetails = new BitSet(); - for(int i=0;i eventList, DataMovementEventPayloadProto.Builder payloadBuilder = DataMovementEventPayloadProto .newBuilder(); - // Construct the VertexManager event if required. if (generateVmEvent) { ShuffleUserPayloads.VertexManagerEventPayloadProto.Builder vmBuilder = @@ -377,7 +376,6 @@ public static void generateEventsForNonStartedOutput(List eventList, DataMovementEventPayloadProto payloadProto = payloadBuilder.build(); ByteBuffer dmePayload = payloadProto.toByteString().asReadOnlyByteBuffer(); - if (isCompositeEvent) { CompositeDataMovementEvent cdme = CompositeDataMovementEvent.create(0, numPhysicalOutputs, dmePayload); @@ -404,9 +402,9 @@ public static void generateEventsForNonStartedOutput(List eventList, * @throws IOException */ public static void generateEventOnSpill(List eventList, boolean finalMergeEnabled, - boolean isLastEvent, OutputContext context, int spillId, TezSpillRecord spillRecord, - int numPhysicalOutputs, boolean sendEmptyPartitionDetails, String pathComponent, - @Nullable long[] partitionStats, boolean reportDetailedPartitionStats, String auxiliaryService, Deflater deflater) + boolean isLastEvent, OutputContext context, int spillId, TezSpillRecord spillRecord, + int numPhysicalOutputs, boolean sendEmptyPartitionDetails, String pathComponent, + @Nullable long[] partitionStats, boolean reportDetailedPartitionStats, String auxiliaryService, Deflater deflater) throws IOException { Objects.requireNonNull(eventList, "EventList can't be null"); @@ -438,8 +436,8 @@ public static void generateEventOnSpill(List eventList, boolean finalMerg } public static VertexManagerEvent generateVMEvent(OutputContext context, - long[] sizePerPartition, boolean reportDetailedPartitionStats, Deflater deflater) - throws IOException { + long[] sizePerPartition, boolean reportDetailedPartitionStats, Deflater deflater) + throws IOException { ShuffleUserPayloads.VertexManagerEventPayloadProto.Builder vmBuilder = ShuffleUserPayloads.VertexManagerEventPayloadProto.newBuilder(); @@ -451,7 +449,7 @@ public static VertexManagerEvent generateVMEvent(OutputContext context, // This is needed for auto-reduce parallelism to work properly. vmBuilder.setOutputSize(outputSize); vmBuilder.setNumRecord(context.getCounters().findCounter(TaskCounter.OUTPUT_RECORDS).getValue() - + context.getCounters().findCounter(TaskCounter.OUTPUT_LARGE_RECORDS).getValue()); + + context.getCounters().findCounter(TaskCounter.OUTPUT_LARGE_RECORDS).getValue()); //set partition stats if (sizePerPartition != null && sizePerPartition.length > 0) { @@ -507,7 +505,7 @@ static long ceil(long a, long b) { getDetailedPartitionStatsForPhysicalOutput(long[] sizes) { DetailedPartitionStatsProto.Builder builder = DetailedPartitionStatsProto.newBuilder(); - for (int i=0; i the value of sizeInMB == 1 // Throws IllegalArgumentException if value is greater than // Integer.MAX_VALUE. That should be ok given Integer.MAX_VALUE * MB @@ -531,7 +529,6 @@ public FetchStatsLogger(Logger activeLogger, Logger aggregateLogger) { this.aggregateLogger = aggregateLogger; } - private static StringBuilder toShortString(InputAttemptIdentifier inputAttemptIdentifier, StringBuilder sb) { sb.append("{"); sb.append(inputAttemptIdentifier.getInputIdentifier()); @@ -545,6 +542,7 @@ private static StringBuilder toShortString(InputAttemptIdentifier inputAttemptId sb.append("}"); return sb; } + /** * Log individual fetch complete event. * This log information would be used by tez-tool/perf-analzyer/shuffle tools for mining @@ -559,7 +557,7 @@ private static StringBuilder toShortString(InputAttemptIdentifier inputAttemptId * @param srcAttemptIdentifier */ public void logIndividualFetchComplete(long millis, long bytesCompressed, - long bytesDecompressed, String outputType, InputAttemptIdentifier srcAttemptIdentifier) { + long bytesDecompressed, String outputType, InputAttemptIdentifier srcAttemptIdentifier) { if (activeLogger.isInfoEnabled()) { long wholeMBs = 0; @@ -605,9 +603,9 @@ public void logIndividualFetchComplete(long millis, long bytesCompressed, } if (currentCount % 1000 == 0) { double avgRate = currentTotalTime == 0 ? 0 - : currentCompressedSize / (double)currentTotalTime / 1000 / 1024 / 1024; + : currentCompressedSize / (double) currentTotalTime / 1000 / 1024 / 1024; aggregateLogger.info("Completed {} fetches, stats for last 1000 fetches: " - + "avg csize: {}, avg dsize: {}, avgTime: {}, avgRate: {}", currentCount, + + "avg csize: {}, avg dsize: {}, avgTime: {}, avgRate: {}", currentCount, currentCompressedSize / 1000, currentDecompressedSize / 1000, currentTotalTime / 1000, MBPS_FORMAT.get().format(avgRate)); } @@ -627,8 +625,7 @@ public static HttpConnectionParams getHttpConnectionParams(Configuration conf) { public static boolean isTezShuffleHandler(Configuration config) { return config.get(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID, - TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID_DEFAULT). + TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID_DEFAULT). contains("tez"); } } - diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java index ca1259f30a..fa0c97aff5 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleInputEventHandlerImpl.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,7 +16,6 @@ * limitations under the License. */ - package org.apache.tez.runtime.library.common.shuffle.impl; import java.io.IOException; @@ -57,7 +56,7 @@ public class ShuffleInputEventHandlerImpl implements ShuffleEventHandler { private static final Logger LOG = LoggerFactory.getLogger(ShuffleInputEventHandlerImpl.class); - + private final ShuffleManager shuffleManager; //TODO: unused. Consider removing later? private final FetchedInputAllocator inputAllocator; @@ -97,11 +96,11 @@ public void handleEvents(List events) throws IOException { handleEvent(event); } } - + private void handleEvent(Event event) throws IOException { if (event instanceof DataMovementEvent) { numDmeEvents.incrementAndGet(); - DataMovementEvent dmEvent = (DataMovementEvent)event; + DataMovementEvent dmEvent = (DataMovementEvent) event; DataMovementEventPayloadProto shufflePayload; try { shufflePayload = DataMovementEventPayloadProto.parseFrom(ByteString.copyFrom(dmEvent.getUserPayload())); @@ -120,7 +119,7 @@ private void handleEvent(Event event) throws IOException { processDataMovementEvent(dmEvent, shufflePayload, emptyPartitionsBitSet); shuffleManager.updateEventReceivedTime(); } else if (event instanceof CompositeRoutedDataMovementEvent) { - CompositeRoutedDataMovementEvent crdme = (CompositeRoutedDataMovementEvent)event; + CompositeRoutedDataMovementEvent crdme = (CompositeRoutedDataMovementEvent) event; DataMovementEventPayloadProto shufflePayload; try { shufflePayload = DataMovementEventPayloadProto.parseFrom(ByteString.copyFrom(crdme.getUserPayload())); @@ -184,7 +183,7 @@ private void processDataMovementEvent(DataMovementEvent dme, DataMovementEventPa CompositeInputAttemptIdentifier srcAttemptIdentifier = constructInputAttemptIdentifier(dme.getTargetIndex(), 1, dme.getVersion(), shufflePayload, false); LOG.debug("Source partition: {} did not generate any data. SrcAttempt: [{}]. Not fetching.", - srcIndex, srcAttemptIdentifier); + srcIndex, srcAttemptIdentifier); numDmeEventsNoData.getAndIncrement(); shuffleManager.addCompletedInputWithNoData(srcAttemptIdentifier.expand(0)); return; @@ -210,34 +209,34 @@ private void processDataMovementEvent(DataMovementEvent dme, DataMovementEventPa LOG.debug("Payload via DME : " + srcAttemptIdentifier); } else { shuffleManager.addKnownInput(shufflePayload.getHost(), shufflePayload.getPort(), - srcAttemptIdentifier, srcIndex); + srcAttemptIdentifier, srcIndex); } } private void moveDataToFetchedInput(DataProto dataProto, - FetchedInput fetchedInput, String hostIdentifier) throws IOException { + FetchedInput fetchedInput, String hostIdentifier) throws IOException { switch (fetchedInput.getType()) { - case DISK: - ShuffleUtils - .shuffleToDisk(((DiskFetchedInput) fetchedInput).getOutputStream(), - hostIdentifier, dataProto.getData().newInput(), - dataProto.getCompressedLength(), - dataProto.getUncompressedLength(), LOG, - fetchedInput.getInputAttemptIdentifier(), ifileReadAhead, - ifileReadAheadLength, true); - break; - case MEMORY: - ShuffleUtils - .shuffleToMemory(((MemoryFetchedInput) fetchedInput).getBytes(), - dataProto.getData().newInput(), dataProto.getRawLength(), - dataProto.getCompressedLength(), - codec, ifileReadAhead, ifileReadAheadLength, LOG, - fetchedInput.getInputAttemptIdentifier()); - break; - case WAIT: - default: - throw new TezUncheckedException("Unexpected type: " - + fetchedInput.getType()); + case DISK: + ShuffleUtils + .shuffleToDisk(((DiskFetchedInput) fetchedInput).getOutputStream(), + hostIdentifier, dataProto.getData().newInput(), + dataProto.getCompressedLength(), + dataProto.getUncompressedLength(), LOG, + fetchedInput.getInputAttemptIdentifier(), ifileReadAhead, + ifileReadAheadLength, true); + break; + case MEMORY: + ShuffleUtils + .shuffleToMemory(((MemoryFetchedInput) fetchedInput).getBytes(), + dataProto.getData().newInput(), dataProto.getRawLength(), + dataProto.getCompressedLength(), + codec, ifileReadAhead, ifileReadAheadLength, LOG, + fetchedInput.getInputAttemptIdentifier()); + break; + case WAIT: + default: + throw new TezUncheckedException("Unexpected type: " + + fetchedInput.getType()); } } @@ -260,7 +259,7 @@ private void processCompositeRoutedDataMovementEvent(CompositeRoutedDataMovement if (emptyPartitionsBitSet.get(srcPartitionId)) { InputAttemptIdentifier srcAttemptIdentifier = compositeInputAttemptIdentifier.expand(i); LOG.debug("Source partition: {} did not generate any data. SrcAttempt: [{}]. Not fetching.", - srcPartitionId, srcAttemptIdentifier); + srcPartitionId, srcAttemptIdentifier); numDmeEventsNoData.getAndIncrement(); shuffleManager.addCompletedInputWithNoData(srcAttemptIdentifier); } @@ -293,7 +292,7 @@ private void processInputFailedEvent(InputFailedEvent ife) { * @return CompositeInputAttemptIdentifier */ private CompositeInputAttemptIdentifier constructInputAttemptIdentifier(int targetIndex, int targetIndexCount, int version, - DataMovementEventPayloadProto shufflePayload, boolean isShared) { + DataMovementEventPayloadProto shufflePayload, boolean isShared) { String pathComponent = (shufflePayload.hasPathComponent()) ? shufflePayload.getPathComponent() : null; CompositeInputAttemptIdentifier srcAttemptIdentifier = null; if (shufflePayload.hasSpillId()) { @@ -309,6 +308,4 @@ private CompositeInputAttemptIdentifier constructInputAttemptIdentifier(int targ } return srcAttemptIdentifier; } - } - diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java index 769ac68f7e..b1cc6c28ee 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -152,7 +152,7 @@ public class ShuffleManager implements FetcherCallback { private final BlockingQueue pendingHosts; private final Set obsoletedInputs; private Set runningFetchers; - + private final AtomicInteger numCompletedInputs = new AtomicInteger(0); private final AtomicInteger numFetchedSpills = new AtomicInteger(0); @@ -163,10 +163,10 @@ public class ShuffleManager implements FetcherCallback { // Required to be held when manipulating pendingHosts private final ReentrantLock lock = new ReentrantLock(); private final Condition wakeLoop = lock.newCondition(); - + private final int numFetchers; private final boolean asyncHttp; - + // Parameters required by Fetchers private final JobTokenSecretManager jobTokenSecretMgr; private final CompressionCodec codec; @@ -184,7 +184,7 @@ public class ShuffleManager implements FetcherCallback { * Holds the time to wait for failures to batch them and send less events. */ private final int maxTimeToWaitForReportMillis; - + private final String sourceDestNameTrimmed; private final int maxTaskOutputAtOnce; @@ -201,10 +201,9 @@ public class ShuffleManager implements FetcherCallback { private final TezCounter bytesShuffledToDiskCounter; private final TezCounter bytesShuffledToMemCounter; private final TezCounter bytesShuffledDirectDiskCounter; - + private volatile Throwable shuffleError; private final HttpConnectionParams httpConnectionParams; - private final LocalDirAllocator localDirAllocator; private final RawLocalFileSystem localFs; @@ -221,10 +220,10 @@ public class ShuffleManager implements FetcherCallback { final Map shuffleInfoEventsMap; // TODO More counters - FetchErrors, speed? - + public ShuffleManager(InputContext inputContext, Configuration conf, int numInputs, - int bufferSize, boolean ifileReadAheadEnabled, int ifileReadAheadLength, - CompressionCodec codec, FetchedInputAllocator inputAllocator) throws IOException { + int bufferSize, boolean ifileReadAheadEnabled, int ifileReadAheadLength, + CompressionCodec codec, FetchedInputAllocator inputAllocator) throws IOException { this.inputContext = inputContext; this.numInputs = numInputs; @@ -237,7 +236,6 @@ public ShuffleManager(InputContext inputContext, Configuration conf, int numInpu this.bytesShuffledToMemCounter = inputContext.getCounters().findCounter(TaskCounter.SHUFFLE_BYTES_TO_MEM); this.bytesShuffledDirectDiskCounter = inputContext.getCounters().findCounter(TaskCounter.SHUFFLE_BYTES_DISK_DIRECT); - this.ifileBufferSize = bufferSize; this.ifileReadAhead = ifileReadAheadEnabled; this.ifileReadAheadLength = ifileReadAheadLength; @@ -254,12 +252,11 @@ public ShuffleManager(InputContext inputContext, Configuration conf, int numInpu TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_BATCH_WAIT, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_BATCH_WAIT_DEFAULT); - this.shufflePhaseTime = inputContext.getCounters().findCounter(TaskCounter.SHUFFLE_PHASE_TIME); this.firstEventReceived = inputContext.getCounters().findCounter(TaskCounter.FIRST_EVENT_RECEIVED); this.lastEventReceived = inputContext.getCounters().findCounter(TaskCounter.LAST_EVENT_RECEIVED); this.compositeFetch = ShuffleUtils.isTezShuffleHandler(conf); - + this.enableFetcherTestingErrors = conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_ENABLE_TESTING_ERRORS, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_ENABLE_TESTING_ERRORS_DEFAULT); @@ -278,11 +275,11 @@ public ShuffleManager(InputContext inputContext, Configuration conf, int numInpu obsoletedInputs = Collections.newSetFromMap(new ConcurrentHashMap()); runningFetchers = Collections.newSetFromMap(new ConcurrentHashMap()); - int maxConfiguredFetchers = + int maxConfiguredFetchers = conf.getInt( TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES_DEFAULT); - + this.numFetchers = Math.min(maxConfiguredFetchers, numInputs); final ExecutorService fetcherRawExecutor; @@ -300,7 +297,7 @@ public ShuffleManager(InputContext inputContext, Configuration conf, int numInpu .setDaemon(true).setNameFormat("ShuffleRunner {" + sourceDestNameTrimmed + "}").build()); this.schedulerExecutor = MoreExecutors.listeningDecorator(schedulerRawExecutor); this.schedulerCallable = new RunShuffleCallable(conf); - + this.startTime = System.currentTimeMillis(); this.lastProgressTime = startTime; @@ -340,7 +337,7 @@ public ShuffleManager(InputContext inputContext, Configuration conf, int numInpu LOG.info(sourceDestNameTrimmed + ": numInputs=" + numInputs + ", compressionCodec=" + (codec == null ? "NoCompressionCodec" : codec.getClass().getName()) + ", numFetchers=" + numFetchers + ", ifileBufferSize=" + ifileBufferSize + ", ifileReadAheadEnabled=" - + ifileReadAhead + ", ifileReadAheadLength=" + ifileReadAheadLength +", " + + ifileReadAhead + ", ifileReadAheadLength=" + ifileReadAheadLength + ", " + "localDiskFetchEnabled=" + localDiskFetchEnabled + ", " + "sharedFetchEnabled=" + sharedFetchEnabled + ", " + httpConnectionParams.toString() + ", maxTaskOutputAtOnce=" + maxTaskOutputAtOnce @@ -378,6 +375,7 @@ private class ReporterCallable extends CallableWithNdc { * Measures if the batching interval has ended. */ private final Clock clock; + ReporterCallable() { clock = new MonotonicClock(); } @@ -460,7 +458,7 @@ protected Void callInternal() throws Exception { } catch (InterruptedException e) { if (isShutdown.get()) { LOG.info(sourceDestNameTrimmed + ": " + - "Interrupted and hasBeenShutdown, Breaking out of ShuffleScheduler Loop"); + "Interrupted and hasBeenShutdown, Breaking out of ShuffleScheduler Loop"); Thread.currentThread().interrupt(); break; } else { @@ -500,7 +498,7 @@ protected Void callInternal() throws Exception { } shufflePhaseTime.setValue(System.currentTimeMillis() - startTime); LOG.info(sourceDestNameTrimmed + ": " + "Shutting down FetchScheduler, Was Interrupted: " + - Thread.currentThread().isInterrupted()); + Thread.currentThread().isInterrupted()); if (!fetcherExecutor.isShutdown()) { fetcherExecutor.shutdownNow(); } @@ -564,8 +562,8 @@ Fetcher constructFetcherForHost(InputHost inputHost, Configuration conf) { .clearAndGetOnePartitionRange(); int includedMaps = 0; for (Iterator inputIter = - pendingInputsOfOnePartitionRange.getInputs().iterator(); - inputIter.hasNext();) { + pendingInputsOfOnePartitionRange.getInputs().iterator(); + inputIter.hasNext(); ) { InputAttemptIdentifier input = inputIter.next(); //For pipelined shuffle. @@ -576,12 +574,12 @@ Fetcher constructFetcherForHost(InputHost inputHost, Configuration conf) { // Avoid adding attempts which have already completed. boolean alreadyCompleted; if (input instanceof CompositeInputAttemptIdentifier) { - CompositeInputAttemptIdentifier compositeInput = (CompositeInputAttemptIdentifier)input; + CompositeInputAttemptIdentifier compositeInput = (CompositeInputAttemptIdentifier) input; int nextClearBit = completedInputSet.nextClearBit(compositeInput.getInputIdentifier()); int maxClearBit = compositeInput.getInputIdentifier() + compositeInput.getInputIdentifierCount(); alreadyCompleted = nextClearBit > maxClearBit; } else { - alreadyCompleted = completedInputSet.get(input.getInputIdentifier()); + alreadyCompleted = completedInputSet.get(input.getInputIdentifier()); } // Avoid adding attempts which have already completed or have been marked as OBSOLETE if (alreadyCompleted || obsoletedInputs.contains(input)) { @@ -602,7 +600,7 @@ Fetcher constructFetcherForHost(InputHost inputHost, Configuration conf) { if (inputHost.getNumPendingPartitions() > 0) { pendingHosts.add(inputHost); //add it to queue } - for(InputAttemptIdentifier input : pendingInputsOfOnePartitionRange.getInputs()) { + for (InputAttemptIdentifier input : pendingInputsOfOnePartitionRange.getInputs()) { ShuffleEventInfo eventInfo = shuffleInfoEventsMap.get(input.getInputIdentifier()); if (eventInfo != null) { eventInfo.scheduledForDownload = true; @@ -611,7 +609,7 @@ Fetcher constructFetcherForHost(InputHost inputHost, Configuration conf) { fetcherBuilder.assignWork(inputHost.getHost(), inputHost.getPort(), pendingInputsOfOnePartitionRange.getPartition(), pendingInputsOfOnePartitionRange.getPartitionCount(), - pendingInputsOfOnePartitionRange.getInputs()); + pendingInputsOfOnePartitionRange.getInputs()); if (LOG.isDebugEnabled()) { LOG.debug("Created Fetcher for host: " + inputHost.getHost() + ", info: " + inputHost.getAdditionalInfo() @@ -619,9 +617,9 @@ Fetcher constructFetcherForHost(InputHost inputHost, Configuration conf) { } return fetcherBuilder.build(); } - + /////////////////// Methods for InputEventHandler - + public void addKnownInput(String hostName, int port, CompositeInputAttemptIdentifier srcAttemptIdentifier, int srcPhysicalIndex) { HostPort identifier = new HostPort(hostName, port); @@ -731,7 +729,7 @@ public void addCompletedInputWithData( } } - protected synchronized void updateEventReceivedTime() { + protected synchronized void updateEventReceivedTime() { long relativeTime = System.currentTimeMillis() - startTime; if (firstEventReceived.getValue() == 0) { firstEventReceived.setValue(relativeTime); @@ -760,7 +758,6 @@ static class ShuffleEventInfo { String id; boolean scheduledForDownload; // whether chunks got scheduled for download - ShuffleEventInfo(InputAttemptIdentifier input) { this.id = input.getInputIdentifier() + "_" + input.getAttemptNumber(); this.eventsProcessed = new BitSet(); @@ -790,14 +787,14 @@ boolean isDone() { public String toString() { return "[eventsProcessed=" + eventsProcessed + ", finalEventId=" + finalEventId - + ", id=" + id + ", attemptNum=" + attemptNum + + ", id=" + id + ", attemptNum=" + attemptNum + ", scheduledForDownload=" + scheduledForDownload + "]"; } } @Override public void fetchSucceeded(String host, InputAttemptIdentifier srcAttemptIdentifier, - FetchedInput fetchedInput, long fetchedBytes, long decompressedLength, long copyDuration) + FetchedInput fetchedInput, long fetchedBytes, long decompressedLength, long copyDuration) throws IOException { int inputIdentifier = srcAttemptIdentifier.getInputIdentifier(); @@ -888,7 +885,7 @@ private void adjustCompletedInputs(FetchedInput fetchedInput) { } private void registerCompletedInputForPipelinedShuffle(InputAttemptIdentifier - srcAttemptIdentifier, FetchedInput fetchedInput) { + srcAttemptIdentifier, FetchedInput fetchedInput) { /** * For pipelinedshuffle it is possible to get multiple spills. Claim success only when * all spills pertaining to an attempt are done. @@ -906,7 +903,7 @@ private void registerCompletedInputForPipelinedShuffle(InputAttemptIdentifier shuffleInfoEventsMap.put(inputIdentifier, eventInfo); } - assert(eventInfo != null); + assert (eventInfo != null); eventInfo.spillProcessed(srcAttemptIdentifier.getSpillEventId()); numFetchedSpills.getAndIncrement(); @@ -923,7 +920,6 @@ private void registerCompletedInputForPipelinedShuffle(InputAttemptIdentifier */ maybeInformInputReady(fetchedInput); - //check if we downloaded all spills pertaining to this InputAttemptIdentifier if (eventInfo.isDone()) { adjustCompletedInputs(fetchedInput); @@ -945,7 +941,7 @@ private void reportFatalError(Throwable exception, String message) { @Override public void fetchFailed(String host, - InputAttemptFetchFailure inputAttemptFetchFailure, boolean connectFailed) { + InputAttemptFetchFailure inputAttemptFetchFailure, boolean connectFailed) { // TODO NEWTEZ. Implement logic to report fetch failures after a threshold. // For now, reporting immediately. InputAttemptIdentifier srcAttemptIdentifier = inputAttemptFetchFailure.getInputAttemptIdentifier(); @@ -1070,7 +1066,6 @@ public float getNumCompletedInputsFloat() { /////////////////// End of methods for walking the available inputs - /** * Fake input that is added to the completed input list in case an input does not have any data. * @@ -1133,11 +1128,9 @@ private void logProgress() { numInputs + ". Transfer rate (CumulativeDataFetched/TimeSinceInputStarted)) " + mbpsFormat.format(transferRate) + " MB/s)"); - } } - private class SchedulerFutureCallback implements FutureCallback { @Override @@ -1154,17 +1147,16 @@ public void onFailure(Throwable t) { inputContext.reportFailure(TaskFailureType.NON_FATAL, t, "Shuffle Scheduler Failed"); } } - } - + private class FetchFutureCallback implements FutureCallback { private final Fetcher fetcher; - + public FetchFutureCallback(Fetcher fetcher) { this.fetcher = fetcher; } - + private void doBookKeepingForFetcherComplete() { lock.lock(); try { @@ -1174,7 +1166,7 @@ private void doBookKeepingForFetcherComplete() { lock.unlock(); } } - + @Override public void onSuccess(FetchResult result) { fetcher.shutdown(); @@ -1212,4 +1204,3 @@ public void onFailure(Throwable t) { } } } - diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/SimpleFetchedInputAllocator.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/SimpleFetchedInputAllocator.java index 6072c039c1..62344acec7 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/SimpleFetchedInputAllocator.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/SimpleFetchedInputAllocator.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -38,7 +38,6 @@ import org.apache.tez.runtime.library.common.shuffle.FetchedInputCallback; import org.apache.tez.runtime.library.common.shuffle.MemoryFetchedInput; - /** * Usage: Create instance, setInitialMemoryAvailable(long), configureAndStart() * @@ -48,7 +47,7 @@ public class SimpleFetchedInputAllocator implements FetchedInputAllocator, FetchedInputCallback { private static final Logger LOG = LoggerFactory.getLogger(SimpleFetchedInputAllocator.class); - + private final Configuration conf; private final TezTaskOutputFiles fileNameAllocator; @@ -62,7 +61,7 @@ public class SimpleFetchedInputAllocator implements FetchedInputAllocator, private final long initialMemoryAvailable; private final String srcNameTrimmed; - + private volatile long usedMemory = 0; public SimpleFetchedInputAllocator(String srcNameTrimmed, @@ -71,14 +70,14 @@ public SimpleFetchedInputAllocator(String srcNameTrimmed, long maxTaskAvailableMemory, long memoryAvailable) { this.srcNameTrimmed = srcNameTrimmed; - this.conf = conf; + this.conf = conf; this.maxAvailableTaskMemory = maxTaskAvailableMemory; this.initialMemoryAvailable = memoryAvailable; - + this.fileNameAllocator = new TezTaskOutputFiles(conf, uniqueIdentifier, dagID); this.localDirAllocator = new LocalDirAllocator(TezRuntimeFrameworkConfigs.LOCAL_DIRS); - + // Setup configuration final float maxInMemCopyUse = conf.getFloat( TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, @@ -88,10 +87,10 @@ public SimpleFetchedInputAllocator(String srcNameTrimmed, + TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT + ": " + maxInMemCopyUse); } - + long memReq = (long) (conf.getLong(Constants.TEZ_RUNTIME_TASK_MEMORY, Math.min(maxAvailableTaskMemory, Integer.MAX_VALUE)) * maxInMemCopyUse); - + if (memReq <= this.initialMemoryAvailable) { this.memoryLimit = memReq; } else { @@ -117,7 +116,6 @@ public SimpleFetchedInputAllocator(String srcNameTrimmed, + ", AssignedMemory=" + this.memoryLimit + ", maxSingleShuffleLimit=" + this.maxSingleShuffleLimit ); - } @Private @@ -137,7 +135,7 @@ public static long getInitialMemoryReq(Configuration conf, long maxAvailableTask @Override public synchronized FetchedInput allocate(long actualSize, long compressedSize, - InputAttemptIdentifier inputAttemptIdentifier) throws IOException { + InputAttemptIdentifier inputAttemptIdentifier) throws IOException { if (actualSize > maxSingleShuffleLimit || this.usedMemory + actualSize > this.memoryLimit) { return new DiskFetchedInput(compressedSize, @@ -155,30 +153,30 @@ public synchronized FetchedInput allocate(long actualSize, long compressedSize, @Override public synchronized FetchedInput allocateType(Type type, long actualSize, - long compressedSize, InputAttemptIdentifier inputAttemptIdentifier) + long compressedSize, InputAttemptIdentifier inputAttemptIdentifier) throws IOException { switch (type) { - case DISK: - return new DiskFetchedInput(compressedSize, - inputAttemptIdentifier, this, conf, localDirAllocator, - fileNameAllocator); - default: - return allocate(actualSize, compressedSize, inputAttemptIdentifier); + case DISK: + return new DiskFetchedInput(compressedSize, + inputAttemptIdentifier, this, conf, localDirAllocator, + fileNameAllocator); + default: + return allocate(actualSize, compressedSize, inputAttemptIdentifier); } } @Override public synchronized void fetchComplete(FetchedInput fetchedInput) { switch (fetchedInput.getType()) { - // Not tracking anything here. - case DISK: - case DISK_DIRECT: - case MEMORY: - break; - default: - throw new TezUncheckedException("InputType: " + fetchedInput.getType() - + " not expected for Broadcast fetch"); + // Not tracking anything here. + case DISK: + case DISK_DIRECT: + case MEMORY: + break; + default: + throw new TezUncheckedException("InputType: " + fetchedInput.getType() + + " not expected for Broadcast fetch"); } } @@ -194,22 +192,21 @@ public synchronized void freeResources(FetchedInput fetchedInput) { private void cleanup(FetchedInput fetchedInput) { switch (fetchedInput.getType()) { - case DISK: - break; - case MEMORY: - unreserve(((MemoryFetchedInput) fetchedInput).getSize()); - break; - default: - throw new TezUncheckedException("InputType: " + fetchedInput.getType() - + " not expected for Broadcast fetch"); + case DISK: + break; + case MEMORY: + unreserve(((MemoryFetchedInput) fetchedInput).getSize()); + break; + default: + throw new TezUncheckedException("InputType: " + fetchedInput.getType() + + " not expected for Broadcast fetch"); } } private synchronized void unreserve(long size) { this.usedMemory -= size; if (LOG.isDebugEnabled()) { - LOG.debug(srcNameTrimmed + ": " + "Used memory after freeing " + size + " : " + usedMemory); + LOG.debug(srcNameTrimmed + ": " + "Used memory after freeing " + size + " : " + usedMemory); } } - } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/package-info.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/package-info.java index 686edff314..c60f7f519b 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/package-info.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/package-info.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,4 +19,4 @@ @Private package org.apache.tez.runtime.library.common.shuffle.impl; -import org.apache.hadoop.classification.InterfaceAudience.Private; \ No newline at end of file +import org.apache.hadoop.classification.InterfaceAudience.Private; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ExceptionReporter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ExceptionReporter.java index 1fa99afe75..3d24adac5a 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ExceptionReporter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ExceptionReporter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,5 +22,6 @@ */ interface ExceptionReporter { void reportException(Throwable t); + void killSelf(Exception exception, String message); } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetchedInputAllocatorOrderedGrouped.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetchedInputAllocatorOrderedGrouped.java index e1456328af..decd4a07f2 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetchedInputAllocatorOrderedGrouped.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetchedInputAllocatorOrderedGrouped.java @@ -24,9 +24,9 @@ public interface FetchedInputAllocatorOrderedGrouped { // TODO TEZ-912 Consolidate this with FetchedInputAllocator. public MapOutput reserve(InputAttemptIdentifier srcAttemptIdentifier, - long requestedSize, - long compressedLength, - int fetcherId) throws IOException; + long requestedSize, + long compressedLength, + int fetcherId) throws IOException; void closeInMemoryFile(MapOutput mapOutput); diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetcherOrderedGrouped.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetcherOrderedGrouped.java index 5887dcb3ca..46cd153b23 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetcherOrderedGrouped.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetcherOrderedGrouped.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -59,7 +59,7 @@ import com.google.common.annotations.VisibleForTesting; class FetcherOrderedGrouped extends CallableWithNdc { - + private static final Logger LOG = LoggerFactory.getLogger(FetcherOrderedGrouped.class); private static final AtomicInteger nextId = new AtomicInteger(0); @@ -108,7 +108,6 @@ class FetcherOrderedGrouped extends CallableWithNdc { private final boolean asyncHttp; private final boolean compositeFetch; - // Initiative value is 0, which means it hasn't retried yet. private long retryStartTime = 0; @@ -218,6 +217,7 @@ public void shutDown() { } private final Object cleanupLock = new Object(); + private void cleanupCurrentConnection(boolean disconnect) { // Synchronizing on cleanupLock to ensure we don't run into a parallel close // Can't synchronize on the main class itself since that would cause the @@ -240,7 +240,7 @@ private void cleanupCurrentConnection(boolean disconnect) { /** * The crux of the matter... - * + * * @param host {@link MapHost} from which we need to * shuffle available map-outputs. */ @@ -255,9 +255,9 @@ protected void copyFromHost(MapHost host) throws IOException { if (srcAttempts.size() == 0) { return; } - if(LOG.isDebugEnabled()) { + if (LOG.isDebugEnabled()) { LOG.debug("Fetcher " + id + " going to fetch from " + host + " for: " - + srcAttempts + ", partition range: " + minPartition + "-" + maxPartition); + + srcAttempts + ", partition range: " + minPartition + "-" + maxPartition); } populateRemainingMap(srcAttempts); // Construct the url and connect @@ -298,8 +298,8 @@ protected void copyFromHost(MapHost host) throws IOException { LOG.debug("Not reporting connection re-establishment failure since fetcher is stopped"); return; } - failedTasks = new InputAttemptFetchFailure[] { - new InputAttemptFetchFailure(getNextRemainingAttempt()) }; + failedTasks = new InputAttemptFetchFailure[]{ + new InputAttemptFetchFailure(getNextRemainingAttempt())}; break; } } @@ -320,7 +320,7 @@ protected void copyFromHost(MapHost host) throws IOException { } private void invokeCopyFailedForFailedTasks(MapHost host, - InputAttemptFetchFailure[] failedTasks) { + InputAttemptFetchFailure[] failedTasks) { if (failedTasks != null && failedTasks.length > 0) { if (stopped) { if (LOG.isDebugEnabled()) { @@ -433,7 +433,7 @@ public String toString() { } protected InputAttemptFetchFailure[] copyMapOutput(MapHost host, DataInputStream input, - InputAttemptIdentifier inputAttemptIdentifier) throws FetcherReadTimeoutException, IOException { + InputAttemptIdentifier inputAttemptIdentifier) throws FetcherReadTimeoutException, IOException { MapOutput mapOutput = null; InputAttemptIdentifier srcAttemptId = null; long decompressedLength = 0; @@ -461,11 +461,11 @@ protected InputAttemptFetchFailure[] copyMapOutput(MapHost host, DataInputStream InputAttemptIdentifier.PATH_PREFIX + ", partition: " + header.forReduce); if (header.mapId.startsWith(ShuffleHandlerError.DISK_ERROR_EXCEPTION.toString())) { //this should be treated as local fetch failure while reporting later - return new InputAttemptFetchFailure[] { - InputAttemptFetchFailure.fromDiskErrorAtSource(getNextRemainingAttempt()) }; + return new InputAttemptFetchFailure[]{ + InputAttemptFetchFailure.fromDiskErrorAtSource(getNextRemainingAttempt())}; } - return new InputAttemptFetchFailure[] { - InputAttemptFetchFailure.fromAttempt(getNextRemainingAttempt()) }; + return new InputAttemptFetchFailure[]{ + InputAttemptFetchFailure.fromAttempt(getNextRemainingAttempt())}; } else { LOG.debug("Already shutdown. Ignoring invalid map id error"); return EMPTY_ATTEMPT_ID_ARRAY; @@ -488,8 +488,8 @@ protected InputAttemptFetchFailure[] copyMapOutput(MapHost host, DataInputStream LOG.warn("Invalid map id ", e); // Don't know which one was bad, so consider this one bad and dont read // the remaining because we dont know where to start reading from. YARN-1773 - return new InputAttemptFetchFailure[] { - new InputAttemptFetchFailure(getNextRemainingAttempt()) }; + return new InputAttemptFetchFailure[]{ + new InputAttemptFetchFailure(getNextRemainingAttempt())}; } else { if (LOG.isDebugEnabled()) { LOG.debug("Already shutdown. Ignoring invalid map id error. Exception: " + @@ -509,8 +509,8 @@ protected InputAttemptFetchFailure[] copyMapOutput(MapHost host, DataInputStream LOG.warn("Was expecting " + srcAttemptId + " but got null"); } assert (srcAttemptId != null); - return new InputAttemptFetchFailure[] { - new InputAttemptFetchFailure(getNextRemainingAttempt()) }; + return new InputAttemptFetchFailure[]{ + new InputAttemptFetchFailure(getNextRemainingAttempt())}; } else { LOG.debug("Already stopped. Ignoring verification failure."); return EMPTY_ATTEMPT_ID_ARRAY; @@ -578,7 +578,7 @@ protected InputAttemptFetchFailure[] copyMapOutput(MapHost host, DataInputStream endTime - startTime, mapOutput, false); } remaining.remove(inputAttemptIdentifier.toString()); - } catch(IOException | InternalError ioe) { + } catch (IOException | InternalError ioe) { if (stopped) { if (LOG.isDebugEnabled()) { LOG.debug("Not reporting fetch failure for exception during data copy: [" @@ -606,8 +606,8 @@ protected InputAttemptFetchFailure[] copyMapOutput(MapHost host, DataInputStream if (srcAttemptId == null) { return InputAttemptFetchFailure.fromAttempts(remaining.values()); } else { - return new InputAttemptFetchFailure[] { - new InputAttemptFetchFailure(srcAttemptId) }; + return new InputAttemptFetchFailure[]{ + new InputAttemptFetchFailure(srcAttemptId)}; } } LOG.warn("Failed to shuffle output of " + srcAttemptId + @@ -615,8 +615,8 @@ protected InputAttemptFetchFailure[] copyMapOutput(MapHost host, DataInputStream // Inform the shuffle-scheduler mapOutput.abort(); - return new InputAttemptFetchFailure[] { - new InputAttemptFetchFailure(srcAttemptId) }; + return new InputAttemptFetchFailure[]{ + new InputAttemptFetchFailure(srcAttemptId)}; } return null; } @@ -651,7 +651,7 @@ private boolean shouldRetry(MapHost host, Throwable ioe) { return false; } } - + /** * Do some basic verification on the input received -- Being defensive * @param compressedLength @@ -662,12 +662,12 @@ private boolean shouldRetry(MapHost host, Throwable ioe) { * @return true/false, based on if the verification succeeded or not */ private boolean verifySanity(long compressedLength, long decompressedLength, - int forReduce, Map remaining, InputAttemptIdentifier srcAttemptId) { + int forReduce, Map remaining, InputAttemptIdentifier srcAttemptId) { if (compressedLength < 0 || decompressedLength < 0) { wrongLengthErrs.increment(1); LOG.warn(logIdentifier + " invalid lengths in map output header: id: " + - srcAttemptId + " len: " + compressedLength + ", decomp len: " + - decompressedLength); + srcAttemptId + " len: " + compressedLength + ", decomp len: " + + decompressedLength); return false; } @@ -682,7 +682,7 @@ private boolean verifySanity(long compressedLength, long decompressedLength, } return true; } - + private InputAttemptIdentifier getNextRemainingAttempt() { if (remaining.size() > 0) { return remaining.values().iterator().next(); @@ -702,7 +702,7 @@ protected void setupLocalDiskFetch(MapHost host) throws InterruptedException { return; } - if(LOG.isDebugEnabled()) { + if (LOG.isDebugEnabled()) { LOG.debug("Fetcher " + id + " going to fetch (local disk) from " + host + " for: " + srcAttempts + ", partition range: " + minPartition + "-" + maxPartition); } @@ -730,7 +730,7 @@ protected void setupLocalDiskFetch(MapHost host) throws InterruptedException { srcAttemptId = scheduler.getIdentifierForFetchedOutput(srcAttemptId.getPathComponent(), reduceId); Path filename = getShuffleInputFileName(srcAttemptId.getPathComponent(), null); TezIndexRecord indexRecord = getIndexRecord(srcAttemptId.getPathComponent(), reduceId); - if(!indexRecord.hasData()) { + if (!indexRecord.hasData()) { continue; } @@ -753,7 +753,6 @@ protected void setupLocalDiskFetch(MapHost host) throws InterruptedException { LOG.debug("Ignoring fetch error during local disk copy since fetcher has already been stopped"); return; } - } } if (!hasFailures) { @@ -763,7 +762,6 @@ protected void setupLocalDiskFetch(MapHost host) throws InterruptedException { } finally { putBackRemainingMapOutputs(host); } - } @VisibleForTesting @@ -808,10 +806,9 @@ void populateRemainingMap(List origlist) { } private String getPathForLocalDir(String suffix) { - if(ShuffleUtils.isTezShuffleHandler(conf)) { + if (ShuffleUtils.isTezShuffleHandler(conf)) { return Constants.DAG_PREFIX + dagId + Path.SEPARATOR + suffix; } return suffix; } } - diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetcherOrderedGroupedWithInjectableErrors.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetcherOrderedGroupedWithInjectableErrors.java index 9c782f6585..186b768ec4 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetcherOrderedGroupedWithInjectableErrors.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/FetcherOrderedGroupedWithInjectableErrors.java @@ -36,13 +36,13 @@ public class FetcherOrderedGroupedWithInjectableErrors extends FetcherOrderedGro private String srcNameTrimmed; public FetcherOrderedGroupedWithInjectableErrors(HttpConnectionParams httpConnectionParams, - ShuffleScheduler scheduler, FetchedInputAllocatorOrderedGrouped allocator, ExceptionReporter exceptionReporter, - JobTokenSecretManager jobTokenSecretMgr, boolean ifileReadAhead, int ifileReadAheadLength, CompressionCodec codec, - Configuration conf, RawLocalFileSystem localFs, boolean localDiskFetchEnabled, String localHostname, - int shufflePort, MapHost mapHost, TezCounter ioErrsCounter, - TezCounter wrongLengthErrsCounter, TezCounter badIdErrsCounter, TezCounter wrongMapErrsCounter, - TezCounter connectionErrsCounter, TezCounter wrongReduceErrsCounter, boolean asyncHttp, - boolean sslShuffle, boolean verifyDiskChecksum, boolean compositeFetch, InputContext inputContext) { + ShuffleScheduler scheduler, FetchedInputAllocatorOrderedGrouped allocator, ExceptionReporter exceptionReporter, + JobTokenSecretManager jobTokenSecretMgr, boolean ifileReadAhead, int ifileReadAheadLength, CompressionCodec codec, + Configuration conf, RawLocalFileSystem localFs, boolean localDiskFetchEnabled, String localHostname, + int shufflePort, MapHost mapHost, TezCounter ioErrsCounter, + TezCounter wrongLengthErrsCounter, TezCounter badIdErrsCounter, TezCounter wrongMapErrsCounter, + TezCounter connectionErrsCounter, TezCounter wrongReduceErrsCounter, boolean asyncHttp, + boolean sslShuffle, boolean verifyDiskChecksum, boolean compositeFetch, InputContext inputContext) { super(httpConnectionParams, scheduler, allocator, exceptionReporter, jobTokenSecretMgr, ifileReadAhead, ifileReadAheadLength, codec, conf, localFs, localDiskFetchEnabled, localHostname, shufflePort, mapHost, ioErrsCounter, wrongLengthErrsCounter, badIdErrsCounter, wrongMapErrsCounter, connectionErrsCounter, diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/InMemoryReader.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/InMemoryReader.java index 41e843219c..0301d7ed7f 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/InMemoryReader.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/InMemoryReader.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -46,15 +46,18 @@ public ByteArrayDataInput(byte buf[], int offset, int length) { public void reset(byte[] input, int start, int length) { this.buf = input; - this.count = start+length; + this.count = start + length; this.mark = start; this.pos = start; } - public byte[] getData() { return buf; } - public int getPosition() { return pos; } - public int getLength() { return count; } - public int getMark() { return mark; } + public byte[] getData() {return buf;} + + public int getPosition() {return pos;} + + public int getLength() {return count;} + + public int getMark() {return mark;} @Override public void readFully(byte[] b) throws IOException { @@ -78,7 +81,7 @@ public boolean readBoolean() throws IOException { @Override public byte readByte() throws IOException { - return (byte)read(); + return (byte) read(); } @Override @@ -140,8 +143,8 @@ public String readUTF() throws IOException { private int originalKeyPos; public InMemoryReader(MergeManager merger, - InputAttemptIdentifier taskAttemptId, byte[] data, int start, - int length) + InputAttemptIdentifier taskAttemptId, byte[] data, int start, + int length) throws IOException { super(null, length - start, null, null, null, false, 0, -1); this.taskAttemptId = taskAttemptId; @@ -177,7 +180,7 @@ public long getLength() { private void dumpOnError() { File dumpFile = new File("../output/" + taskAttemptId + ".dump"); System.err.println("Dumping corrupt map-output of " + taskAttemptId + - " to " + dumpFile.getAbsolutePath()); + " to " + dumpFile.getAbsolutePath()); FileOutputStream fos = null; try { fos = new FileOutputStream(dumpFile); diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/InMemoryWriter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/InMemoryWriter.java index c5db7c9ade..fdb7851062 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/InMemoryWriter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/InMemoryWriter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -58,7 +58,7 @@ public InMemoryWriter(BoundedByteArrayOutputStream arrayStream, boolean rle) { public void append(Object key, Object value) throws IOException { throw new UnsupportedOperationException - ("InMemoryWriter.append(K key, V value"); + ("InMemoryWriter.append(K key, V value"); } public void close() throws IOException { @@ -74,5 +74,4 @@ public void close() throws IOException { out.close(); out = null; } - } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapHost.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapHost.java index 4746306950..b9e11236f9 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapHost.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapHost.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -89,7 +89,7 @@ public String toString() { private final int partitionCount; // Tracks attempt IDs private List maps = new ArrayList(); - + public MapHost(String host, int port, int partition, int partitionCount) { this.host = host; this.port = port; @@ -133,15 +133,15 @@ public synchronized List getAndClearKnownMaps() { maps = new ArrayList(); return currentKnownMaps; } - + public synchronized void markBusy() { state = State.BUSY; } - + public synchronized void markPenalized() { state = State.PENALIZED; } - + public synchronized int getNumKnownMapOutputs() { return maps.size(); } @@ -158,12 +158,12 @@ public synchronized State markAvailable() { } return state; } - + @Override public String toString() { return getHostIdentifier(); } - + /** * Mark the host as penalized */ diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapOutput.java index 9239a1f5a2..ded7374dd5 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapOutput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MapOutput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -32,11 +32,10 @@ import org.apache.tez.runtime.library.common.InputAttemptIdentifier; import org.apache.tez.runtime.library.common.task.local.output.TezTaskOutputFiles; - abstract class MapOutput { private static final Logger LOG = LoggerFactory.getLogger(MapOutput.class); private static AtomicInteger ID = new AtomicInteger(0); - + public enum Type { WAIT, MEMORY, @@ -79,15 +78,15 @@ public static MapOutput createDiskMapOutput(InputAttemptIdentifier attemptIdenti } public static MapOutput createLocalDiskMapOutput(InputAttemptIdentifier attemptIdentifier, - FetchedInputAllocatorOrderedGrouped callback, Path path, long offset, - long size, boolean primaryMapOutput) { + FetchedInputAllocatorOrderedGrouped callback, Path path, long offset, + long size, boolean primaryMapOutput) { return new DiskDirectMapOutput(attemptIdentifier, callback, size, path, offset, primaryMapOutput); } public static MapOutput createMemoryMapOutput(InputAttemptIdentifier attemptIdentifier, FetchedInputAllocatorOrderedGrouped callback, int size, - boolean primaryMapOutput) { + boolean primaryMapOutput) { return new InMemoryMapOutput(attemptIdentifier, callback, size, primaryMapOutput); } @@ -102,7 +101,7 @@ public boolean isPrimaryMapOutput() { @Override public boolean equals(Object obj) { if (obj instanceof MapOutput) { - return id == ((MapOutput)obj).id; + return id == ((MapOutput) obj).id; } return false; } @@ -119,7 +118,7 @@ public FileChunk getOutputPath() { public byte[] getMemory() { return null; } - + public OutputStream getDisk() { return null; } @@ -136,27 +135,27 @@ public long getSize() { public void commit() throws IOException { } - + public void abort() { } - + public String toString() { return "MapOutput( AttemptIdentifier: " + attemptIdentifier + ", Type: " + getType() + ")"; } - - public static class MapOutputComparator - implements Comparator { + + public static class MapOutputComparator + implements Comparator { public int compare(MapOutput o1, MapOutput o2) { - if (o1.id == o2.id) { + if (o1.id == o2.id) { return 0; } - + if (o1.getSize() < o2.getSize()) { return -1; } else if (o1.getSize() > o2.getSize()) { return 1; } - + if (o1.id < o2.id) { return -1; } else { @@ -167,8 +166,9 @@ public int compare(MapOutput o1, MapOutput o2) { private static class DiskDirectMapOutput extends MapOutput { private final FileChunk outputPath; + private DiskDirectMapOutput(InputAttemptIdentifier attemptIdentifier, FetchedInputAllocatorOrderedGrouped callback, - long size, Path outputPath, long offset, boolean primaryMapOutput) { + long size, Path outputPath, long offset, boolean primaryMapOutput) { super(attemptIdentifier, callback, primaryMapOutput); this.outputPath = new FileChunk(outputPath, offset, size, true, attemptIdentifier); } @@ -203,8 +203,9 @@ private static class DiskMapOutput extends MapOutput { private final Path tmpOutputPath; private final FileChunk outputPath; private OutputStream disk; + private DiskMapOutput(InputAttemptIdentifier attemptIdentifier, FetchedInputAllocatorOrderedGrouped callback, - long size, Path outputPath, long offset, boolean primaryMapOutput, Path tmpOutputPath) { + long size, Path outputPath, long offset, boolean primaryMapOutput, Path tmpOutputPath) { super(attemptIdentifier, callback, primaryMapOutput); this.tmpOutputPath = tmpOutputPath; @@ -250,11 +251,12 @@ public Type getType() { private static class InMemoryMapOutput extends MapOutput { private byte[] byteArray; + private InMemoryMapOutput(InputAttemptIdentifier attemptIdentifier, FetchedInputAllocatorOrderedGrouped callback, long size, boolean primaryMapOutput) { super(attemptIdentifier, callback, primaryMapOutput); - this.byteArray = new byte[(int)size]; + this.byteArray = new byte[(int) size]; } @Override diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java index 8d3f37dbcd..2cec1d1491 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -69,47 +69,46 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; - /** * Usage. Create instance. setInitialMemoryAvailable(long), configureAndStart() * */ @InterfaceAudience.Private @InterfaceStability.Unstable -@SuppressWarnings(value={"rawtypes"}) +@SuppressWarnings(value = {"rawtypes"}) public class MergeManager implements FetchedInputAllocatorOrderedGrouped { - + private static final Logger LOG = LoggerFactory.getLogger(MergeManager.class); private final Configuration conf; private final FileSystem localFS; private final FileSystem rfs; private final LocalDirAllocator localDirAllocator; - - private final TezTaskOutputFiles mapOutputFile; + + private final TezTaskOutputFiles mapOutputFile; private final Progressable progressable = new Progressable() { @Override public void progress() { inputContext.notifyProgress(); } }; - private final Combiner combiner; - + private final Combiner combiner; + @VisibleForTesting final Set inMemoryMergedMapOutputs = - new TreeSet(new MapOutput.MapOutputComparator()); + new TreeSet(new MapOutput.MapOutputComparator()); private final IntermediateMemoryToMemoryMerger memToMemMerger; @VisibleForTesting final Set inMemoryMapOutputs = - new TreeSet(new MapOutput.MapOutputComparator()); + new TreeSet(new MapOutput.MapOutputComparator()); private final InMemoryMerger inMemoryMerger; @VisibleForTesting final Set onDiskMapOutputs = new TreeSet(); @VisibleForTesting final OnDiskMerger onDiskMerger; - + private final long memoryLimit; @VisibleForTesting final long postMergeMemLimit; @@ -120,13 +119,13 @@ public void progress() { private final AtomicBoolean isShutdown = new AtomicBoolean(false); - private final int memToMemMergeOutputsThreshold; + private final int memToMemMergeOutputsThreshold; private final long mergeThreshold; - + private final long initialMemoryAvailable; private final ExceptionReporter exceptionReporter; - + private final InputContext inputContext; private final TezCounter spilledRecordsCounter; @@ -134,16 +133,16 @@ public void progress() { private final TezCounter reduceCombineInputCounter; private final TezCounter mergedMapOutputsCounter; - + private final TezCounter numMemToDiskMerges; private final TezCounter numDiskToDiskMerges; private final TezCounter additionalBytesWritten; private final TezCounter additionalBytesRead; - + private final CompressionCodec codec; - + private volatile boolean finalMergeComplete = false; - + private final boolean ifileReadAhead; private final int ifileReadAheadLength; private final int ifileBufferSize; @@ -153,7 +152,6 @@ public void progress() { private final SegmentStatsTracker statsInMemTotal = new SegmentStatsTracker(); private final SegmentStatsTracker statsInMemLastLog = new SegmentStatsTracker(); - private AtomicInteger mergeFileSequenceId = new AtomicInteger(0); private final boolean cleanup; @@ -165,7 +163,7 @@ public void progress() { */ public MergeManager(Configuration conf, FileSystem localFS, - LocalDirAllocator localDirAllocator, + LocalDirAllocator localDirAllocator, InputContext inputContext, Combiner combiner, TezCounter spilledRecordsCounter, @@ -181,7 +179,7 @@ public MergeManager(Configuration conf, this.localDirAllocator = localDirAllocator; this.exceptionReporter = exceptionReporter; this.initialMemoryAvailable = initialMemoryAvailable; - + this.combiner = combiner; this.reduceCombineInputCounter = reduceCombineInputCounter; @@ -192,8 +190,8 @@ public MergeManager(Configuration conf, inputContext.getDagIdentifier()); this.localFS = localFS; - this.rfs = ((LocalFileSystem)localFS).getRaw(); - + this.rfs = ((LocalFileSystem) localFS).getRaw(); + this.numDiskToDiskMerges = inputContext.getCounters().findCounter(TaskCounter.NUM_DISK_TO_DISK_MERGES); this.numMemToDiskMerges = inputContext.getCounters().findCounter(TaskCounter.NUM_MEM_TO_DISK_MERGES); this.additionalBytesWritten = inputContext.getCounters().findCounter(TaskCounter.ADDITIONAL_SPILLS_BYTES_WRITTEN); @@ -211,12 +209,12 @@ public MergeManager(Configuration conf, } this.ifileBufferSize = conf.getInt("io.file.buffer.size", TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_BUFFER_SIZE_DEFAULT); - + // Figure out initial memory req start final float maxInMemCopyUse = - conf.getFloat( - TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, - TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT_DEFAULT); + conf.getFloat( + TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, + TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT_DEFAULT); if (maxInMemCopyUse > 1.0 || maxInMemCopyUse < 0.0) { throw new IllegalArgumentException("Invalid value for " + TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT + ": " + @@ -224,7 +222,7 @@ public MergeManager(Configuration conf, } // Allow unit tests to fix Runtime memory - long memLimit = conf.getLong(Constants.TEZ_RUNTIME_TASK_MEMORY, (long)(inputContext + long memLimit = conf.getLong(Constants.TEZ_RUNTIME_TASK_MEMORY, (long) (inputContext .getTotalMemoryAvailableToTask() * maxInMemCopyUse)); float maxRedPer = conf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, @@ -235,13 +233,13 @@ public MergeManager(Configuration conf, long maxRedBuffer = (long) (inputContext.getTotalMemoryAvailableToTask() * maxRedPer); // Figure out initial memory req end - + if (this.initialMemoryAvailable < memLimit) { this.memoryLimit = this.initialMemoryAvailable; } else { this.memoryLimit = memLimit; } - + if (this.initialMemoryAvailable < maxRedBuffer) { this.postMergeMemLimit = this.initialMemoryAvailable; } else { @@ -257,11 +255,11 @@ public MergeManager(Configuration conf, + this.memoryLimit + ", postMergeMem: " + this.postMergeMemLimit); } - this.ioSortFactor = + this.ioSortFactor = conf.getInt( - TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, + TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR_DEFAULT); - + final float singleShuffleMemoryLimitPercent = conf.getFloat( TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, @@ -274,31 +272,31 @@ public MergeManager(Configuration conf, } //TODO: Cap it to MAX_VALUE until MapOutput starts supporting > 2 GB - this.maxSingleShuffleLimit = + this.maxSingleShuffleLimit = (long) Math.min((memoryLimit * singleShuffleMemoryLimitPercent), Integer.MAX_VALUE); - this.memToMemMergeOutputsThreshold = - conf.getInt( - TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS, - ioSortFactor); - this.mergeThreshold = - (long)(this.memoryLimit * - conf.getFloat( - TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, - TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT_DEFAULT)); + this.memToMemMergeOutputsThreshold = + conf.getInt( + TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMTOMEM_SEGMENTS, + ioSortFactor); + this.mergeThreshold = + (long) (this.memoryLimit * + conf.getFloat( + TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT, + TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT_DEFAULT)); LOG.info(inputContext.getInputOutputVertexNames() + ": MergerManager: memoryLimit=" + memoryLimit + ", " + - "maxSingleShuffleLimit=" + maxSingleShuffleLimit + ", " + - "mergeThreshold=" + mergeThreshold + ", " + - "ioSortFactor=" + ioSortFactor + ", " + - "postMergeMem=" + postMergeMemLimit + ", " + - "memToMemMergeOutputsThreshold=" + memToMemMergeOutputsThreshold); - + "maxSingleShuffleLimit=" + maxSingleShuffleLimit + ", " + + "mergeThreshold=" + mergeThreshold + ", " + + "ioSortFactor=" + ioSortFactor + ", " + + "postMergeMem=" + postMergeMemLimit + ", " + + "memToMemMergeOutputsThreshold=" + memToMemMergeOutputsThreshold); + if (this.maxSingleShuffleLimit >= this.mergeThreshold) { throw new RuntimeException("Invlaid configuration: " + "maxSingleShuffleLimit should be less than mergeThreshold" + "maxSingleShuffleLimit: " + this.maxSingleShuffleLimit + ", mergeThreshold: " + this.mergeThreshold); } - + boolean allowMemToMemMerge = conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_ENABLE_MEMTOMEM_DEFAULT); @@ -322,7 +320,8 @@ void setupParentThread(Thread shuffleSchedulerThread) { if (this.memToMemMerger != null) { memToMemMerger.setParentThread(shuffleSchedulerThread); } - this.inMemoryMerger.setParentThread(shuffleSchedulerThread);; + this.inMemoryMerger.setParentThread(shuffleSchedulerThread); + ; this.onDiskMerger.setParentThread(shuffleSchedulerThread); } @@ -342,31 +341,31 @@ void configureAndStart() { static long getInitialMemoryRequirement(Configuration conf, long maxAvailableTaskMemory) { final float maxInMemCopyUse = conf.getFloat( - TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, + TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT_DEFAULT); - if (maxInMemCopyUse > 1.0 || maxInMemCopyUse < 0.0) { - throw new IllegalArgumentException("Invalid value for " + - TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT + ": " + - maxInMemCopyUse); - } + if (maxInMemCopyUse > 1.0 || maxInMemCopyUse < 0.0) { + throw new IllegalArgumentException("Invalid value for " + + TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT + ": " + + maxInMemCopyUse); + } - // Allow unit tests to fix Runtime memory - long memLimit = conf.getLong(Constants.TEZ_RUNTIME_TASK_MEMORY, - (long)(maxAvailableTaskMemory * maxInMemCopyUse)); - - float maxRedPer = conf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, - TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_BUFFER_PERCENT_DEFAULT); - if (maxRedPer > 1.0 || maxRedPer < 0.0) { - throw new TezUncheckedException(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT + maxRedPer); - } - long maxRedBuffer = (long) (maxAvailableTaskMemory * maxRedPer); + // Allow unit tests to fix Runtime memory + long memLimit = conf.getLong(Constants.TEZ_RUNTIME_TASK_MEMORY, + (long) (maxAvailableTaskMemory * maxInMemCopyUse)); + + float maxRedPer = conf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, + TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_BUFFER_PERCENT_DEFAULT); + if (maxRedPer > 1.0 || maxRedPer < 0.0) { + throw new TezUncheckedException(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT + maxRedPer); + } + long maxRedBuffer = (long) (maxAvailableTaskMemory * maxRedPer); LOG.info("Initial Memory required for SHUFFLE_BUFFER=" + memLimit + " based on INPUT_BUFFER_FACTOR=" + maxInMemCopyUse + ", for final merged output=" + maxRedBuffer + ", using factor: " + maxRedPer); - long reqMem = Math.max(maxRedBuffer, memLimit); - return reqMem; + long reqMem = Math.max(maxRedBuffer, memLimit); + return reqMem; } public void waitForInMemoryMerge() throws InterruptedException { @@ -398,7 +397,7 @@ private boolean canShuffleToMemory(long requestedSize) { public synchronized void waitForShuffleToMergeMemory() throws InterruptedException { long startTime = System.currentTimeMillis(); - while(usedMemory > memoryLimit) { + while (usedMemory > memoryLimit) { wait(); } if (LOG.isDebugEnabled()) { @@ -410,11 +409,11 @@ public synchronized void waitForShuffleToMergeMemory() throws InterruptedExcepti final private MapOutput stallShuffle = MapOutput.createWaitMapOutput(null); @Override - public synchronized MapOutput reserve(InputAttemptIdentifier srcAttemptIdentifier, - long requestedSize, - long compressedLength, - int fetcher - ) throws IOException { + public synchronized MapOutput reserve(InputAttemptIdentifier srcAttemptIdentifier, + long requestedSize, + long compressedLength, + int fetcher + ) throws IOException { if (!canShuffleToMemory(requestedSize)) { if (LOG.isDebugEnabled()) { LOG.debug(srcAttemptIdentifier + ": Shuffling to disk since " + requestedSize + @@ -424,7 +423,7 @@ public synchronized MapOutput reserve(InputAttemptIdentifier srcAttemptIdentifie return MapOutput.createDiskMapOutput(srcAttemptIdentifier, this, compressedLength, conf, fetcher, true, mapOutputFile); } - + // Stall shuffle if we are above the memory limit // It is possible that all threads could just be stalling and not make @@ -439,7 +438,7 @@ public synchronized MapOutput reserve(InputAttemptIdentifier srcAttemptIdentifie // (usedMemory + requestedSize > memoryLimit). When this thread is done // fetching, this will automatically trigger a merge thereby unlocking // all the stalled threads - + if (usedMemory > memoryLimit) { if (LOG.isDebugEnabled()) { LOG.debug(srcAttemptIdentifier + ": Stalling shuffle since usedMemory (" + usedMemory @@ -448,7 +447,7 @@ public synchronized MapOutput reserve(InputAttemptIdentifier srcAttemptIdentifie } return stallShuffle; } - + // Allow the in-memory shuffle to progress if (LOG.isDebugEnabled()) { LOG.debug(srcAttemptIdentifier + ": Proceeding with shuffle since usedMemory (" @@ -457,7 +456,7 @@ public synchronized MapOutput reserve(InputAttemptIdentifier srcAttemptIdentifie } return unconditionalReserve(srcAttemptIdentifier, requestedSize, true); } - + /** * Unconditional Reserve is used by the Memory-to-Memory thread */ @@ -465,7 +464,7 @@ private synchronized MapOutput unconditionalReserve( InputAttemptIdentifier srcAttemptIdentifier, long requestedSize, boolean primaryMapOutput) throws IOException { usedMemory += requestedSize; - return MapOutput.createMemoryMapOutput(srcAttemptIdentifier, this, (int)requestedSize, + return MapOutput.createMemoryMapOutput(srcAttemptIdentifier, this, (int) requestedSize, primaryMapOutput); } @@ -485,13 +484,12 @@ public synchronized void releaseCommittedMemory(long size) { unreserve(size); } - @Override - public synchronized void closeInMemoryFile(MapOutput mapOutput) { + public synchronized void closeInMemoryFile(MapOutput mapOutput) { inMemoryMapOutputs.add(mapOutput); trackAndLogCloseInMemoryFile(mapOutput); - commitMemory+= mapOutput.getSize(); + commitMemory += mapOutput.getSize(); if (commitMemory >= mergeThreshold) { startMemToDiskMerge(); @@ -500,7 +498,7 @@ public synchronized void closeInMemoryFile(MapOutput mapOutput) { // This should likely run a Combiner. if (memToMemMerger != null) { synchronized (memToMemMerger) { - if (!memToMemMerger.isInProgress() && + if (!memToMemMerger.isInProgress() && inMemoryMapOutputs.size() >= memToMemMergeOutputsThreshold) { memToMemMerger.startMerge(inMemoryMapOutputs); } @@ -559,7 +557,7 @@ private void startMemToDiskMerge() { } } } - + public synchronized void closeInMemoryMergedFile(MapOutput mapOutput) { inMemoryMergedMapOutputs.add(mapOutput); if (LOG.isDebugEnabled()) { @@ -590,8 +588,8 @@ public synchronized void closeOnDiskFile(FileChunk file) { //ensure offsets are not the same. Preconditions.checkArgument(fileChunk.getOffset() != file.getOffset(), "Can't have a file with same path and offset." - + "OldFilePath=" + fileChunk.getPath() + ", OldFileOffset=" + fileChunk.getOffset() + - ", newFilePath=" + file.getPath() + ", newFileOffset=" + file.getOffset()); + + "OldFilePath=" + fileChunk.getPath() + ", OldFileOffset=" + fileChunk.getOffset() + + ", newFilePath=" + file.getPath() + ", newFileOffset=" + file.getOffset()); } } @@ -607,6 +605,7 @@ public synchronized void closeOnDiskFile(FileChunk file) { } private long lastOnDiskSegmentLogTime = -1L; + private void logCloseOnDiskFile(FileChunk file) { if (LOG.isDebugEnabled()) { LOG.debug( @@ -627,7 +626,7 @@ private void logCloseOnDiskFile(FileChunk file) { * Should only be used after the Shuffle phaze is complete, otherwise can * return an invalid state since a merge may not be in progress dur to * inadequate inputs - * + * * @return true if the merge process is complete, otherwise false */ @Private @@ -686,7 +685,6 @@ public boolean isShutdown() { return isShutdown.get(); } - static void cleanup(FileSystem fs, Collection fileChunkList) { for (FileChunk fileChunk : fileChunkList) { cleanup(fs, fileChunk.getPath()); @@ -706,8 +704,6 @@ static void cleanup(FileSystem fs, Path path) { } } - - void runCombineProcessor(TezRawKeyValueIterator kvIter, Writer writer) throws IOException, InterruptedException { combiner.combine(kvIter, writer); @@ -716,10 +712,10 @@ void runCombineProcessor(TezRawKeyValueIterator kvIter, Writer writer) /** * Merges multiple in-memory segment to another in-memory segment */ - private class IntermediateMemoryToMemoryMerger - extends MergeThread { - - public IntermediateMemoryToMemoryMerger(MergeManager manager, + private class IntermediateMemoryToMemoryMerger + extends MergeThread { + + public IntermediateMemoryToMemoryMerger(MergeManager manager, int mergeFactor) { super(manager, mergeFactor, exceptionReporter); setName("MemToMemMerger [" + TezUtilsInternal @@ -736,7 +732,7 @@ public void merge(List inputs) throws IOException, InterruptedExcepti inputContext.notifyProgress(); - InputAttemptIdentifier dummyMapId = inputs.get(0).getAttemptIdentifier(); + InputAttemptIdentifier dummyMapId = inputs.get(0).getAttemptIdentifier(); List inMemorySegments = new ArrayList(); MapOutput mergedMapOutputs = null; @@ -747,7 +743,7 @@ public void merge(List inputs) throws IOException, InterruptedExcepti Iterator it = inputs.iterator(); MapOutput lastAddedMapOutput = null; - while(it.hasNext() && !Thread.currentThread().isInterrupted()) { + while (it.hasNext() && !Thread.currentThread().isInterrupted()) { MapOutput mo = it.next(); if ((mergeOutputSize + mo.getSize() + manager.getUsedMemory()) > memoryLimit) { //Search for smaller segments that can fit into existing mem @@ -799,19 +795,19 @@ public void merge(List inputs) throws IOException, InterruptedExcepti // Nothing will be materialized to disk because the sort factor is being // set to the number of in memory segments. // TODO Is this doing any combination ? - TezRawKeyValueIterator rIter = - TezMerger.merge(conf, rfs, - serializationContext, - inMemorySegments, inMemorySegments.size(), - new Path(inputContext.getUniqueIdentifier()), - (RawComparator)ConfigUtils.getIntermediateInputKeyComparator(conf), - progressable, null, null, null, null); + TezRawKeyValueIterator rIter = + TezMerger.merge(conf, rfs, + serializationContext, + inMemorySegments, inMemorySegments.size(), + new Path(inputContext.getUniqueIdentifier()), + (RawComparator) ConfigUtils.getIntermediateInputKeyComparator(conf), + progressable, null, null, null, null); TezMerger.writeFile(rIter, writer, progressable, TezRuntimeConfiguration.TEZ_RUNTIME_RECORDS_BEFORE_PROGRESS_DEFAULT); writer.close(); LOG.info(inputContext.getInputOutputVertexNames() + - " Memory-to-Memory merge of the " + noInMemorySegments + - " files in-memory complete with mergeOutputSize=" + mergeOutputSize); + " Memory-to-Memory merge of the " + noInMemorySegments + + " files in-memory complete with mergeOutputSize=" + mergeOutputSize); // Note the output of the merge closeInMemoryMergedFile(mergedMapOutputs); @@ -823,7 +819,7 @@ public void cleanup(List inputs, boolean deleteData) throws IOExcepti //No OP } } - + /** * Merges multiple in-memory segment to a disk segment */ @@ -840,12 +836,12 @@ private class InMemoryMerger extends MergeThread { public InMemoryMerger(MergeManager manager) { super(manager, Integer.MAX_VALUE, exceptionReporter); - setName("MemtoDiskMerger [" + TezUtilsInternal + setName("MemtoDiskMerger [" + TezUtilsInternal .cleanVertexName(inputContext.getSourceVertexName()) - + "_" + inputContext.getUniqueIdentifier() + "]"); + + "_" + inputContext.getUniqueIdentifier() + "]"); setDaemon(true); } - + @Override public void merge(List inputs) throws IOException, InterruptedException { if (inputs == null || inputs.size() == 0) { @@ -854,7 +850,7 @@ public void merge(List inputs) throws IOException, InterruptedExcepti numMemToDiskMerges.increment(1); inputContext.notifyProgress(); - + //name this output file same as the name of the first file that is //there in the current list of inmem files (this is guaranteed to //be absent on the disk currently. So we don't overwrite a prev. @@ -868,11 +864,11 @@ public void merge(List inputs) throws IOException, InterruptedExcepti List inMemorySegments = new ArrayList(); long mergeOutputSize = - createInMemorySegments(inputs, inMemorySegments,0); + createInMemorySegments(inputs, inMemorySegments, 0); int noInMemorySegments = inMemorySegments.size(); // TODO Maybe track serialized vs deserialized bytes. - + // All disk writes done by this merge are overhead - due to the lack of // adequate memory to keep all segments in memory. outputPath = mapOutputFile.getInputFileForWrite( @@ -889,7 +885,7 @@ public void merge(List inputs) throws IOException, InterruptedExcepti null, null); TezRawKeyValueIterator rIter = null; - LOG.info("Initiating in-memory merge with " + noInMemorySegments + + LOG.info("Initiating in-memory merge with " + noInMemorySegments + " segments..."); tmpDir = new Path(inputContext.getUniqueIdentifier()); @@ -897,7 +893,7 @@ public void merge(List inputs) throws IOException, InterruptedExcepti rIter = TezMerger.merge(conf, rfs, serializationContext, inMemorySegments, inMemorySegments.size(), - tmpDir, (RawComparator)ConfigUtils.getIntermediateInputKeyComparator(conf), + tmpDir, (RawComparator) ConfigUtils.getIntermediateInputKeyComparator(conf), progressable, spilledRecordsCounter, null, additionalBytesRead, null); // spilledRecordsCounter is tracking the number of keys that will be // read from each of the segments being merged - which is essentially @@ -919,7 +915,7 @@ public void merge(List inputs) throws IOException, InterruptedExcepti " files in-memory complete." + " Local file is " + outputPath + " of size " + outFileLen); - } catch (IOException e) { + } catch (IOException e) { //make sure that we delete the ondisk file that we created //earlier when we invoked cloneFileAttributes localFS.delete(outputPath, true); @@ -961,12 +957,12 @@ class OnDiskMerger extends MergeThread { public OnDiskMerger(MergeManager manager) { super(manager, ioSortFactor, exceptionReporter); - setName("DiskToDiskMerger [" + TezUtilsInternal + setName("DiskToDiskMerger [" + TezUtilsInternal .cleanVertexName(inputContext.getSourceVertexName()) + "_" + inputContext.getUniqueIdentifier() + "]"); setDaemon(true); } - + @Override public void merge(List inputs) throws IOException, InterruptedException { // sanity check @@ -978,11 +974,11 @@ public void merge(List inputs) throws IOException, InterruptedExcepti inputContext.notifyProgress(); long approxOutputSize = 0; - int bytesPerSum = - conf.getInt("io.bytes.per.checksum", 512); - - LOG.info("OnDiskMerger: We have " + inputs.size() + - " map outputs on disk. Triggering merge..."); + int bytesPerSum = + conf.getInt("io.bytes.per.checksum", 512); + + LOG.info("OnDiskMerger: We have " + inputs.size() + + " map outputs on disk. Triggering merge..."); List inputSegments = new ArrayList(inputs.size()); @@ -1004,8 +1000,8 @@ public void merge(List inputs) throws IOException, InterruptedExcepti } // add the checksum length - approxOutputSize += - ChecksumFileSystem.getChecksumLength(approxOutputSize, bytesPerSum); + approxOutputSize += + ChecksumFileSystem.getChecksumLength(approxOutputSize, bytesPerSum); // 2. Start the on-disk merge process FileChunk file0 = inputs.get(0); @@ -1034,7 +1030,7 @@ public void merge(List inputs) throws IOException, InterruptedExcepti serializationContext, inputSegments, ioSortFactor, tmpDir, - (RawComparator)ConfigUtils.getIntermediateInputKeyComparator(conf), + (RawComparator) ConfigUtils.getIntermediateInputKeyComparator(conf), progressable, true, spilledRecordsCounter, null, mergedMapOutputsCounter, null); @@ -1053,9 +1049,9 @@ public void merge(List inputs) throws IOException, InterruptedExcepti closeOnDiskFile(new FileChunk(outputPath, 0, outputLen)); LOG.info(inputContext.getInputOutputVertexNames() + - " Finished merging " + inputs.size() + - " map output files on disk of total-size " + - approxOutputSize + "." + + " Finished merging " + inputs.size() + + " map output files on disk of total-size " + + approxOutputSize + "." + " Local output file is " + outputPath + " of size " + outputLen); } @@ -1074,11 +1070,11 @@ public void cleanup(List inputs, boolean deleteData) throws IOExcepti } } } - + private long createInMemorySegments(List inMemoryMapOutputs, - List inMemorySegments, + List inMemorySegments, long leaveBytes - ) throws IOException { + ) throws IOException { long totalSize = 0L; // We could use fullSize could come from the RamManager, but files can be // closed but not yet present in inMemoryMapOutputs @@ -1087,18 +1083,18 @@ private long createInMemorySegments(List inMemoryMapOutputs, fullSize += mo.getSize(); } int inMemoryMapOutputsOffset = 0; - while((fullSize > leaveBytes) && !Thread.currentThread().isInterrupted()) { + while ((fullSize > leaveBytes) && !Thread.currentThread().isInterrupted()) { MapOutput mo = inMemoryMapOutputs.get(inMemoryMapOutputsOffset++); byte[] data = mo.getMemory(); long size = data.length; totalSize += size; fullSize -= size; - IFile.Reader reader = new InMemoryReader(MergeManager.this, - mo.getAttemptIdentifier(), - data, 0, (int)size); + IFile.Reader reader = new InMemoryReader(MergeManager.this, + mo.getAttemptIdentifier(), + data, 0, (int) size); inMemorySegments.add(new Segment(reader, - (mo.isPrimaryMapOutput() ? - mergedMapOutputsCounter : null))); + (mo.isPrimaryMapOutput() ? + mergedMapOutputsCounter : null))); } // Bulk remove removed in-memory map outputs efficiently inMemoryMapOutputs.subList(0, inMemoryMapOutputsOffset).clear(); @@ -1117,6 +1113,7 @@ public RawKVIteratorReader(TezRawKeyValueIterator kvIter, long size) this.kvIter = kvIter; this.size = size; } + @Override public KeyState readRawKey(DataInputBuffer key) throws IOException { if (kvIter.next()) { @@ -1129,6 +1126,7 @@ public KeyState readRawKey(DataInputBuffer key) throws IOException { } return KeyState.NO_KEY; } + public void nextRawValue(DataInputBuffer value) throws IOException { final DataInputBuffer vb = kvIter.getValue(); final int vp = vb.getPosition(); @@ -1136,6 +1134,7 @@ public void nextRawValue(DataInputBuffer value) throws IOException { value.reset(vb.getData(), vp, vlen); bytesRead += vlen; } + public long getPosition() throws IOException { return bytesRead; } @@ -1144,26 +1143,27 @@ public void close() throws IOException { kvIter.close(); } - @Override public long getLength() { + @Override + public long getLength() { return size; } } private TezRawKeyValueIterator finalMerge(Configuration job, FileSystem fs, - List inMemoryMapOutputs, - List onDiskMapOutputs - ) throws IOException, InterruptedException { + List inMemoryMapOutputs, + List onDiskMapOutputs + ) throws IOException, InterruptedException { logFinalMergeStart(inMemoryMapOutputs, onDiskMapOutputs); StringBuilder finalMergeLog = new StringBuilder(); - + inputContext.notifyProgress(); // merge config params SerializationContext serContext = new SerializationContext(job); final Path tmpDir = new Path(inputContext.getUniqueIdentifier()); final RawComparator comparator = - (RawComparator)ConfigUtils.getIntermediateInputKeyComparator(job); + (RawComparator) ConfigUtils.getIntermediateInputKeyComparator(job); // segments required to vacate memory List memDiskSegments = new ArrayList(); @@ -1172,26 +1172,26 @@ private TezRawKeyValueIterator finalMerge(Configuration job, FileSystem fs, if (inMemoryMapOutputs.size() > 0) { int srcTaskId = inMemoryMapOutputs.get(0).getAttemptIdentifier().getInputIdentifier(); inMemToDiskBytes = createInMemorySegments(inMemoryMapOutputs, - memDiskSegments, - this.postMergeMemLimit); + memDiskSegments, + this.postMergeMemLimit); final int numMemDiskSegments = memDiskSegments.size(); if (numMemDiskSegments > 0 && - ioSortFactor > onDiskMapOutputs.size()) { - + ioSortFactor > onDiskMapOutputs.size()) { + // If we reach here, it implies that we have less than io.sort.factor // disk segments and this will be incremented by 1 (result of the // memory segments merge). Since this total would still be // <= io.sort.factor, we will not do any more intermediate merges, // the merge of all these disk segments would be directly fed to the // reduce method - + mergePhaseFinished = true; // must spill to disk, but can't retain in-mem for intermediate merge // Can not use spill id in final merge as it would clobber with other files, hence using // Integer.MAX_VALUE - final Path outputPath = - mapOutputFile.getInputFileForWrite(srcTaskId, Integer.MAX_VALUE, - inMemToDiskBytes).suffix(Constants.MERGED_OUTPUT_PREFIX); + final Path outputPath = + mapOutputFile.getInputFileForWrite(srcTaskId, Integer.MAX_VALUE, + inMemToDiskBytes).suffix(Constants.MERGED_OUTPUT_PREFIX); final TezRawKeyValueIterator rIter = TezMerger.merge(job, fs, serContext, memDiskSegments, numMemDiskSegments, tmpDir, comparator, progressable, spilledRecordsCounter, null, additionalBytesRead, null); @@ -1262,7 +1262,7 @@ private TezRawKeyValueIterator finalMerge(Configuration job, FileSystem fs, final long fileOffset = fileChunk.getOffset(); final boolean preserve = fileChunk.isLocalFile(); diskSegments.add(new DiskSegment(fs, file, fileOffset, fileLength, codec, ifileReadAhead, - ifileReadAheadLength, ifileBufferSize, preserve, counter)); + ifileReadAheadLength, ifileBufferSize, preserve, counter)); } if (LOG.isInfoEnabled()) { finalMergeLog.append(". DiskSeg: " + onDisk.length + ", " + onDiskBytes); @@ -1282,8 +1282,8 @@ public int compare(Segment o1, Segment o2) { // build final list of segments from merged backed by disk + in-mem List finalSegments = new ArrayList(); - long inMemBytes = createInMemorySegments(inMemoryMapOutputs, - finalSegments, 0); + long inMemBytes = createInMemorySegments(inMemoryMapOutputs, + finalSegments, 0); if (LOG.isInfoEnabled()) { finalMergeLog.append(". MemSeg: " + finalSegments.size() + ", " + inMemBytes); if (LOG.isDebugEnabled()) { @@ -1305,7 +1305,7 @@ public int compare(Segment o1, Segment o2) { return diskMerge; } finalSegments.add(new Segment( - new RawKVIteratorReader(diskMerge, onDiskBytes), null)); + new RawKVIteratorReader(diskMerge, onDiskBytes), null)); } if (LOG.isInfoEnabled()) { LOG.info(finalMergeLog.toString()); @@ -1317,7 +1317,6 @@ public int compare(Segment o1, Segment o2) { additionalBytesRead, null); } - private void logFinalMergeStart(List inMemoryMapOutputs, List onDiskMapOutputs) { long inMemSegmentSize = 0; @@ -1343,7 +1342,6 @@ private void logFinalMergeStart(List inMemoryMapOutputs, "finalMerge with #inMemoryOutputs={}, size={} and #onDiskOutputs={}, size={}", inMemoryMapOutputs.size(), inMemSegmentSize, onDiskMapOutputs.size(), onDiskSegmentSize); - } @VisibleForTesting @@ -1361,8 +1359,6 @@ void waitForMemToMemMerge() throws InterruptedException { memToMemMerger.waitForMerge(); } - - private static class SegmentStatsTracker { private long size; private int count; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeThread.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeThread.java index c0af90fde0..9e1702370d 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeThread.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeThread.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -27,7 +27,7 @@ import org.slf4j.LoggerFactory; abstract class MergeThread extends Thread { - + private static final Logger LOG = LoggerFactory.getLogger(MergeThread.class); private volatile boolean inProgress = false; @@ -38,14 +38,14 @@ abstract class MergeThread extends Thread { private final int mergeFactor; private Thread shuffleSchedulerThread; - + public MergeThread(MergeManager manager, int mergeFactor, ExceptionReporter reporter) { this.manager = manager; this.mergeFactor = mergeFactor; this.reporter = reporter; } - + public synchronized void close() throws InterruptedException { closed = true; if (!Thread.currentThread().isInterrupted()) { @@ -69,18 +69,18 @@ public void setParentThread(Thread shuffleSchedulerThread) { public synchronized boolean isInProgress() { return inProgress; } - + public synchronized void startMerge(Set inputs) { if (!closed) { this.inputs.clear(); inProgress = true; - Iterator iter=inputs.iterator(); + Iterator iter = inputs.iterator(); for (int ctr = 0; iter.hasNext() && ctr < mergeFactor; ++ctr) { this.inputs.add(iter.next()); iter.remove(); } - LOG.info(getName() + ": Starting merge with " + this.inputs.size() + - " segments, while ignoring " + inputs.size() + " segments"); + LOG.info(getName() + ": Starting merge with " + this.inputs.size() + + " segments, while ignoring " + inputs.size() + " segments"); notifyAll(); } } @@ -115,21 +115,21 @@ public void run() { // Meant to handle a shutdown of the entire fetch/merge process Thread.currentThread().interrupt(); return; - } catch(Throwable t) { + } catch (Throwable t) { reporter.reportException(t); return; } finally { synchronized (this) { // Clear inputs inputs.clear(); - inProgress = false; + inProgress = false; notifyAll(); } } } } - public abstract void merge(List inputs) + public abstract void merge(List inputs) throws IOException, InterruptedException; public abstract void cleanup(List inputs, boolean deleteData) diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/Shuffle.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/Shuffle.java index 2b99739a58..01d421f725 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/Shuffle.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/Shuffle.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,7 +26,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; - import com.google.common.annotations.VisibleForTesting; import org.apache.tez.runtime.api.TaskFailureType; import org.apache.tez.runtime.library.common.shuffle.ShuffleUtils; @@ -69,13 +68,13 @@ @InterfaceAudience.Private @InterfaceStability.Unstable public class Shuffle implements ExceptionReporter { - + private static final Logger LOG = LoggerFactory.getLogger(Shuffle.class); private static final int PROGRESS_FREQUENCY = 2000; - + private final Configuration conf; private final InputContext inputContext; - + private final ShuffleInputEventHandlerOrderedGrouped eventHandler; @VisibleForTesting final ShuffleScheduler scheduler; @@ -92,9 +91,9 @@ public class Shuffle implements ExceptionReporter { private final RunShuffleCallable runShuffleCallable; private volatile ListenableFuture runShuffleFuture; private final ListeningExecutorService executor; - + private final String sourceDestNameTrimmed; - + private AtomicBoolean isShutDown = new AtomicBoolean(false); private AtomicBoolean fetchersClosed = new AtomicBoolean(false); private AtomicBoolean schedulerClosed = new AtomicBoolean(false); @@ -105,13 +104,13 @@ public class Shuffle implements ExceptionReporter { private final TezCounter shufflePhaseTime; public Shuffle(InputContext inputContext, Configuration conf, int numInputs, - long initialMemoryAvailable) throws IOException { + long initialMemoryAvailable) throws IOException { this.inputContext = inputContext; this.conf = conf; this.sourceDestNameTrimmed = TezUtilsInternal.cleanVertexName(inputContext.getSourceVertexName()) + " -> " + TezUtilsInternal.cleanVertexName(inputContext.getTaskVertexName()); - + this.codec = CodecUtils.getCodec(conf); this.ifileReadAhead = conf.getBoolean( @@ -124,11 +123,11 @@ public Shuffle(InputContext inputContext, Configuration conf, int numInputs, } else { this.ifileReadAheadLength = 0; } - + Combiner combiner = TezRuntimeUtils.instantiateCombiner(conf, inputContext); - + FileSystem localFS = FileSystem.getLocal(this.conf); - LocalDirAllocator localDirAllocator = + LocalDirAllocator localDirAllocator = new LocalDirAllocator(TezRuntimeFrameworkConfigs.LOCAL_DIRS); // TODO TEZ Get rid of Map / Reduce references. @@ -160,32 +159,29 @@ public Shuffle(InputContext inputContext, Configuration conf, int numInputs, ifileReadAheadLength); scheduler = new ShuffleScheduler( - this.inputContext, - this.conf, - numInputs, - this, - merger, - merger, - startTime, - codec, - ifileReadAhead, - ifileReadAheadLength, - sourceDestNameTrimmed); + this.inputContext, + this.conf, + numInputs, + this, + merger, + merger, + startTime, + codec, + ifileReadAhead, + ifileReadAheadLength, + sourceDestNameTrimmed); this.mergePhaseTime = inputContext.getCounters().findCounter(TaskCounter.MERGE_PHASE_TIME); this.shufflePhaseTime = inputContext.getCounters().findCounter(TaskCounter.SHUFFLE_PHASE_TIME); - - - eventHandler= new ShuffleInputEventHandlerOrderedGrouped( + eventHandler = new ShuffleInputEventHandlerOrderedGrouped( inputContext, scheduler, ShuffleUtils.isTezShuffleHandler(conf)); - + ExecutorService rawExecutor = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder() .setDaemon(true).setNameFormat("ShuffleAndMergeRunner {" + sourceDestNameTrimmed + "}").build()); - executor = MoreExecutors.listeningDecorator(rawExecutor); runShuffleCallable = new RunShuffleCallable(); } @@ -196,15 +192,14 @@ public void handleEvents(List events) throws IOException { } else { LOG.info(sourceDestNameTrimmed + ": " + "Ignoring events since already shutdown. EventCount: " + events.size()); } - } - + /** * Indicates whether the Shuffle and Merge processing is complete. * @return false if not complete, true if complete or if an error occurred. - * @throws InterruptedException - * @throws IOException - * @throws InputAlreadyClosedException + * @throws InterruptedException + * @throws IOException + * @throws InputAlreadyClosedException */ public boolean isInputReady() throws IOException, InterruptedException, TezException { if (isShutDown.get()) { @@ -313,13 +308,13 @@ protected TezRawKeyValueIterator callInternal() throws IOException, InterruptedE throw new ShuffleError("Error while doing final merge ", e); } mergePhaseTime.setValue(System.currentTimeMillis() - startTime); - + inputContext.notifyProgress(); // Sanity check synchronized (Shuffle.this) { if (throwable.get() != null) { throw new ShuffleError("error in shuffle in " + throwingThreadName, - throwable.get()); + throwable.get()); } } @@ -335,7 +330,7 @@ private void cleanupShuffleSchedulerIgnoreErrors() { } catch (InterruptedException e) { Thread.currentThread().interrupt(); LOG.info(sourceDestNameTrimmed + ": " + - "Interrupted while attempting to close the scheduler during cleanup. Ignoring"); + "Interrupted while attempting to close the scheduler during cleanup. Ignoring"); } } @@ -354,7 +349,7 @@ private void cleanupMerger(boolean ignoreErrors) throws Throwable { //Reset the status Thread.currentThread().interrupt(); LOG.info(sourceDestNameTrimmed + ": " + - "Interrupted while attempting to close the merger during cleanup. Ignoring"); + "Interrupted while attempting to close the merger during cleanup. Ignoring"); } else { throw e; } @@ -409,7 +404,7 @@ public synchronized void killSelf(Exception exception, String message) { inputContext.killSelf(exception, message); } } - + public static class ShuffleError extends IOException { private static final long serialVersionUID = 5753909320586607881L; @@ -422,7 +417,7 @@ public static class ShuffleError extends IOException { public static long getInitialMemoryRequirement(Configuration conf, long maxAvailableTaskMemory) { return MergeManager.getInitialMemoryRequirement(conf, maxAvailableTaskMemory); } - + private class ShuffleRunnerFutureCallback implements FutureCallback { @Override public void onSuccess(TezRawKeyValueIterator result) { diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleHeader.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleHeader.java index f074e897e8..db7ed04f50 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleHeader.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleHeader.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -35,7 +35,7 @@ @InterfaceAudience.Private @InterfaceStability.Stable public class ShuffleHeader implements Writable { - + /** Header info of the shuffle http request/response */ public static final String HTTP_HEADER_NAME = "name"; public static final String DEFAULT_HTTP_HEADER_NAME = "mapreduce"; @@ -51,25 +51,25 @@ public class ShuffleHeader implements Writable { long uncompressedLength; long compressedLength; int forReduce; - - public ShuffleHeader() { } - + + public ShuffleHeader() {} + public ShuffleHeader(String mapId, long compressedLength, - long uncompressedLength, int forReduce) { + long uncompressedLength, int forReduce) { this.mapId = mapId; this.compressedLength = compressedLength; this.uncompressedLength = uncompressedLength; this.forReduce = forReduce; } - + public String getMapId() { return this.mapId; } - + public int getPartition() { return this.forReduce; } - + public long getUncompressedLength() { return uncompressedLength; } @@ -97,6 +97,7 @@ public int writeLength() throws IOException { return length; } + public void write(DataOutput out) throws IOException { Text.writeString(out, mapId); WritableUtils.writeVLong(out, compressedLength); diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java index 9984c5af10..6da93c90f9 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleInputEventHandlerOrderedGrouped.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -46,7 +46,7 @@ import com.google.protobuf.InvalidProtocolBufferException; public class ShuffleInputEventHandlerOrderedGrouped implements ShuffleEventHandler { - + private static final Logger LOG = LoggerFactory.getLogger(ShuffleInputEventHandlerOrderedGrouped.class); private final ShuffleScheduler scheduler; @@ -87,7 +87,7 @@ public void logProgress(boolean updateOnClose) { private void handleEvent(Event event) throws IOException { if (event instanceof DataMovementEvent) { numDmeEvents.incrementAndGet(); - DataMovementEvent dmEvent = (DataMovementEvent)event; + DataMovementEvent dmEvent = (DataMovementEvent) event; DataMovementEventPayloadProto shufflePayload; try { shufflePayload = DataMovementEventPayloadProto.parseFrom(ByteString.copyFrom(dmEvent.getUserPayload())); @@ -106,7 +106,7 @@ private void handleEvent(Event event) throws IOException { processDataMovementEvent(dmEvent, shufflePayload, emptyPartitionsBitSet); scheduler.updateEventReceivedTime(); } else if (event instanceof CompositeRoutedDataMovementEvent) { - CompositeRoutedDataMovementEvent crdme = (CompositeRoutedDataMovementEvent)event; + CompositeRoutedDataMovementEvent crdme = (CompositeRoutedDataMovementEvent) event; DataMovementEventPayloadProto shufflePayload; try { shufflePayload = DataMovementEventPayloadProto.parseFrom(ByteString.copyFrom(crdme.getUserPayload())); @@ -157,7 +157,7 @@ private void processDataMovementEvent(DataMovementEvent dmEvent, DataMovementEve try { if (emptyPartitionsBitSet.get(partitionId)) { LOG.debug("Source partition: {} did not generate any data. SrcAttempt: [{}]. Not fetching.", - partitionId, srcAttemptIdentifier); + partitionId, srcAttemptIdentifier); numDmeEventsNoData.getAndIncrement(); scheduler.copySucceeded(srcAttemptIdentifier.expand(0), null, 0, 0, 0, null, true); return; @@ -239,4 +239,3 @@ private CompositeInputAttemptIdentifier constructInputAttemptIdentifier(int targ return srcAttemptIdentifier; } } - diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleScheduler.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleScheduler.java index 470b04cc5f..3df5614ed3 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleScheduler.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/ShuffleScheduler.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -141,6 +141,7 @@ enum ShuffleErrors { CONNECTION, WRONG_REDUCE } + @VisibleForTesting final static String SHUFFLE_ERR_GRP_NAME = "Shuffle Errors"; @@ -177,10 +178,10 @@ enum ShuffleErrors { private final DelayQueue penalties = new DelayQueue(); private final Referee referee; @VisibleForTesting - final Map failureCounts = new HashMap(); + final Map failureCounts = new HashMap(); final Set uniqueHosts = Sets.newHashSet(); - private final Map hostFailures = new HashMap(); + private final Map hostFailures = new HashMap(); private final InputContext inputContext; private final TezCounter shuffledInputsCounter; private final TezCounter skippedInputCounter; @@ -251,7 +252,7 @@ enum ShuffleErrors { private final int maxPenaltyTime; private long totalBytesShuffledTillNow = 0; - private final DecimalFormat mbpsFormat = new DecimalFormat("0.00"); + private final DecimalFormat mbpsFormat = new DecimalFormat("0.00"); public ShuffleScheduler(InputContext inputContext, Configuration conf, @@ -304,7 +305,7 @@ public ShuffleScheduler(InputContext inputContext, + "=" + minFailurePerHost + " should not be negative"); this.hostFailureFraction = conf.getFloat(TezRuntimeConfiguration - .TEZ_RUNTIME_SHUFFLE_ACCEPTABLE_HOST_FETCH_FAILURE_FRACTION, + .TEZ_RUNTIME_SHUFFLE_ACCEPTABLE_HOST_FETCH_FAILURE_FRACTION, TezRuntimeConfiguration .TEZ_RUNTIME_SHUFFLE_ACCEPTABLE_HOST_FETCH_FAILURE_FRACTION_DEFAULT); @@ -332,7 +333,7 @@ public ShuffleScheduler(InputContext inputContext, this.checkFailedFetchSinceLastCompletion = conf.getBoolean (TezRuntimeConfiguration - .TEZ_RUNTIME_SHUFFLE_FAILED_CHECK_SINCE_LAST_COMPLETION, + .TEZ_RUNTIME_SHUFFLE_FAILED_CHECK_SINCE_LAST_COMPLETION, TezRuntimeConfiguration .TEZ_RUNTIME_SHUFFLE_FAILED_CHECK_SINCE_LAST_COMPLETION_DEFAULT); @@ -354,7 +355,7 @@ public ShuffleScheduler(InputContext inputContext, TaskCounter.NUM_FAILED_SHUFFLE_INPUTS); this.bytesShuffledToDisk = inputContext.getCounters().findCounter( TaskCounter.SHUFFLE_BYTES_TO_DISK); - this.bytesShuffledToDiskDirect = inputContext.getCounters().findCounter(TaskCounter.SHUFFLE_BYTES_DISK_DIRECT); + this.bytesShuffledToDiskDirect = inputContext.getCounters().findCounter(TaskCounter.SHUFFLE_BYTES_DISK_DIRECT); this.bytesShuffledToMem = inputContext.getCounters().findCounter(TaskCounter.SHUFFLE_BYTES_TO_MEM); // Counters used by Fetchers @@ -396,13 +397,13 @@ public ShuffleScheduler(InputContext inputContext, this.maxFailedUniqueFetches = Math.min(numberOfInputs, 5); referee.start(); - this.maxFetchFailuresBeforeReporting = + this.maxFetchFailuresBeforeReporting = conf.getInt( TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_FAILURES_LIMIT_DEFAULT); - this.reportReadErrorImmediately = + this.reportReadErrorImmediately = conf.getBoolean( - TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR, + TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR, TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_NOTIFY_READERROR_DEFAULT); this.verifyDiskChecksum = conf.getBoolean( TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_VERIFY_DISK_CHECKSUM, @@ -413,9 +414,9 @@ public ShuffleScheduler(InputContext inputContext, * be approximately 48 bytes; 48 * 75 = 3600 which should give some room for other info in URL. */ this.maxTaskOutputAtOnce = Math.max(1, Math.min(75, conf.getInt( - TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_MAX_TASK_OUTPUT_AT_ONCE, - TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_MAX_TASK_OUTPUT_AT_ONCE_DEFAULT))); - + TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_MAX_TASK_OUTPUT_AT_ONCE, + TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_MAX_TASK_OUTPUT_AT_ONCE_DEFAULT))); + this.skippedInputCounter = inputContext.getCounters().findCounter(TaskCounter.NUM_SKIPPED_INPUTS); this.firstEventReceived = inputContext.getCounters().findCounter(TaskCounter.FIRST_EVENT_RECEIVED); this.lastEventReceived = inputContext.getCounters().findCounter(TaskCounter.LAST_EVENT_RECEIVED); @@ -526,7 +527,7 @@ public boolean isShutdown() { return isShutdown.get(); } - protected synchronized void updateEventReceivedTime() { + protected synchronized void updateEventReceivedTime() { long relativeTime = System.currentTimeMillis() - startTime; if (firstEventReceived.getValue() == 0) { firstEventReceived.setValue(relativeTime); @@ -547,7 +548,6 @@ static class ShuffleEventInfo { String id; boolean scheduledForDownload; // whether chunks got scheduled for download (getMapHost) - ShuffleEventInfo(InputAttemptIdentifier input) { this.id = input.getInputIdentifier() + "_" + input.getAttemptNumber(); this.eventsProcessed = new BitSet(); @@ -573,7 +573,7 @@ boolean isDone() { public String toString() { return "[eventsProcessed=" + eventsProcessed + ", finalEventId=" + finalEventId - + ", id=" + id + ", attemptNum=" + attemptNum + + ", id=" + id + ", attemptNum=" + attemptNum + ", scheduledForDownload=" + scheduledForDownload + "]"; } } @@ -585,7 +585,7 @@ public synchronized void copySucceeded(InputAttemptIdentifier srcAttemptIdentifi long millis, MapOutput output, boolean isLocalFetch - ) throws IOException { + ) throws IOException { inputContext.notifyProgress(); if (!isInputFinished(srcAttemptIdentifier.getInputIdentifier())) { @@ -643,7 +643,7 @@ public synchronized void copySucceeded(InputAttemptIdentifier srcAttemptIdentifi pipelinedShuffleInfoEventsMap.put(inputIdentifier, eventInfo); } - assert(eventInfo != null); + assert (eventInfo != null); eventInfo.spillProcessed(srcAttemptIdentifier.getSpillEventId()); numFetchedSpills++; @@ -681,15 +681,15 @@ public synchronized void copySucceeded(InputAttemptIdentifier srcAttemptIdentifi if (LOG.isDebugEnabled()) { LOG.debug("src task: " + TezRuntimeUtils.getTaskAttemptIdentifier( - inputContext.getSourceVertexName(), srcAttemptIdentifier.getInputIdentifier(), - srcAttemptIdentifier.getAttemptNumber()) + " done"); + inputContext.getSourceVertexName(), srcAttemptIdentifier.getInputIdentifier(), + srcAttemptIdentifier.getAttemptNumber()) + " done"); } } else { // input is already finished. duplicate fetch. LOG.warn("Duplicate fetch of input no longer needs to be fetched: " + srcAttemptIdentifier); // free the resource - specially memory - + // If the src does not generate data, output will be null. if (output != null) { output.abort(); @@ -760,7 +760,7 @@ private void logProgress() { } public synchronized void copyFailed(InputAttemptFetchFailure fetchFailure, MapHost host, - boolean readError, boolean connectError) { + boolean readError, boolean connectError) { failedShuffleCounter.increment(1); inputContext.notifyProgress(); int failures = incrementAndGetFailureAttempt(fetchFailure.getInputAttemptIdentifier()); @@ -883,7 +883,7 @@ private void informAM(InputAttemptFetchFailure fetchFailure) { InputReadErrorEvent.create( "Fetch failure for " + TezRuntimeUtils.getTaskAttemptIdentifier(inputContext.getSourceVertexName(), - srcAttempt.getInputIdentifier(), srcAttempt.getAttemptNumber()) + srcAttempt.getInputIdentifier(), srcAttempt.getAttemptNumber()) + " to jobtracker.", srcAttempt.getInputIdentifier(), srcAttempt.getAttemptNumber(), fetchFailure.isLocalFetch(), fetchFailure.isDiskErrorAtSource(), localHostname)); @@ -906,7 +906,7 @@ private boolean hasFailedAcrossNodes(String logContext) { (int) Math.ceil(numUniqueHosts * hostFailureFraction)); int total = 0; boolean failedAcrossNodes = false; - for(HostPort host : uniqueHosts) { + for (HostPort host : uniqueHosts) { IntWritable failures = hostFailures.get(host); if (failures != null && failures.get() > minFailurePerHost) { total++; @@ -922,7 +922,7 @@ private boolean hasFailedAcrossNodes(String logContext) { + ", hostFailuresCount=" + hostFailures.size() + ", hosts crossing threshold=" + total + ", reducerFetchIssues=" + failedAcrossNodes - ); + ); return failedAcrossNodes; } @@ -1021,24 +1021,24 @@ boolean isShuffleHealthy(InputAttemptFetchFailure fetchFailure) { String logContext = "srcAttempt=" + srcAttempt.toString(); boolean fetcherHealthy = isFetcherHealthy(logContext); - + // check if the reducer has progressed enough boolean reducerProgressedEnough = - (((float)doneMaps / numInputs) - >= MIN_REQUIRED_PROGRESS_PERCENT); + (((float) doneMaps / numInputs) + >= MIN_REQUIRED_PROGRESS_PERCENT); // check if the reducer is stalled for a long time // duration for which the reducer is stalled int stallDuration = - (int)(System.currentTimeMillis() - lastProgressTime); - + (int) (System.currentTimeMillis() - lastProgressTime); + // duration for which the reducer ran with progress int shuffleProgressDuration = - (int)(lastProgressTime - startTime); + (int) (lastProgressTime - startTime); boolean reducerStalled = (shuffleProgressDuration > 0) && - (((float)stallDuration / shuffleProgressDuration) - >= MAX_ALLOWED_STALL_TIME_PERCENT); + (((float) stallDuration / shuffleProgressDuration) + >= MAX_ALLOWED_STALL_TIME_PERCENT); // kill if not healthy and has insufficient progress if ((failureCounts.size() >= maxFailedUniqueFetches || @@ -1095,7 +1095,7 @@ public synchronized void addKnownMapOutput(String inputHostName, notifyAll(); } } - + public void obsoleteInput(InputAttemptIdentifier srcAttempt) { // The incoming srcAttempt does not contain a path component. LOG.info(srcNameTrimmed + ": " + "Adding obsolete input: " + srcAttempt); @@ -1122,7 +1122,7 @@ public void obsoleteInput(InputAttemptIdentifier srcAttempt) { obsoleteInputs.add(srcAttempt); } } - + public synchronized void putBackKnownMapOutput(MapHost host, InputAttemptIdentifier srcAttempt) { host.addKnownMap(srcAttempt); @@ -1169,7 +1169,7 @@ DelayQueue getPenalties() { private synchronized boolean inputShouldBeConsumed(InputAttemptIdentifier id) { boolean isInputFinished = false; if (id instanceof CompositeInputAttemptIdentifier) { - CompositeInputAttemptIdentifier cid = (CompositeInputAttemptIdentifier)id; + CompositeInputAttemptIdentifier cid = (CompositeInputAttemptIdentifier) id; isInputFinished = isInputFinished(cid.getInputIdentifier(), cid.getInputIdentifier() + cid.getInputIdentifierCount()); } else { isInputFinished = isInputFinished(id.getInputIdentifier()); @@ -1201,7 +1201,7 @@ public synchronized List getMapsForHost(MapHost host) { // more than one item in the oldIdList. boolean addIdentifierToList = false; Iterator oldIdIterator = oldIdList.iterator(); - while(oldIdIterator.hasNext()) { + while (oldIdIterator.hasNext()) { InputAttemptIdentifier oldId = oldIdIterator.next(); //no need to add if spill ids are same @@ -1242,7 +1242,7 @@ public synchronized List getMapsForHost(MapHost host) { int includedMaps = 0; int totalSize = dedupedList.size(); - for(Integer inputIndex : dedupedList.keySet()) { + for (Integer inputIndex : dedupedList.keySet()) { List attemptIdentifiers = dedupedList.get(inputIndex); for (InputAttemptIdentifier inputAttemptIdentifier : attemptIdentifiers) { if (includedMaps++ >= maxTaskOutputAtOnce) { @@ -1300,7 +1300,7 @@ public synchronized boolean isDone() { static class Penalty implements Delayed { MapHost host; private long endTime; - + Penalty(MapHost host, long delay) { this.host = host; this.endTime = System.currentTimeMillis() + delay; @@ -1315,7 +1315,6 @@ public int compareTo(Delayed o) { long other = ((Penalty) o).endTime; return endTime == other ? 0 : (endTime < other ? -1 : 1); } - } /** @@ -1349,10 +1348,9 @@ public void run() { } } } - void setInputFinished(int inputIndex) { - synchronized(finishedMaps) { + synchronized (finishedMaps) { finishedMaps.set(inputIndex, true); } } @@ -1371,7 +1369,6 @@ boolean isInputFinished(int inputIndex, int inputEnd) { private class ShuffleSchedulerCallable extends CallableWithNdc { - @Override protected Void callInternal() throws InterruptedException { while (!isShutdown.get() && remainingMaps.get() > 0) { @@ -1461,8 +1458,8 @@ protected Void callInternal() throws InterruptedException { } private synchronized void waitAndNotifyProgress() throws InterruptedException { - inputContext.notifyProgress(); - wait(1000); + inputContext.notifyProgress(); + wait(1000); } @VisibleForTesting @@ -1500,8 +1497,6 @@ private void doBookKeepingForFetcherComplete() { } } - - @Override public void onSuccess(Void result) { fetcherOrderedGrouped.shutDown(); diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/package-info.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/package-info.java index 1d3210144b..20173e88b8 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/package-info.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/package-info.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,4 +19,4 @@ @Private package org.apache.tez.runtime.library.common.shuffle.orderedgrouped; -import org.apache.hadoop.classification.InterfaceAudience.Private; \ No newline at end of file +import org.apache.hadoop.classification.InterfaceAudience.Private; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/package-info.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/package-info.java index 974932e11d..9291afdc9d 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/package-info.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/package-info.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,4 +19,4 @@ @Private package org.apache.tez.runtime.library.common.shuffle; -import org.apache.hadoop.classification.InterfaceAudience.Private; \ No newline at end of file +import org.apache.hadoop.classification.InterfaceAudience.Private; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java index 758c069799..3f17eaab60 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.runtime.library.common.sort.impl; @@ -84,7 +84,7 @@ public List close() throws IOException { public void write(Object key, Iterable values) throws IOException { //TODO: Sorter classes should override this method later. Iterator it = values.iterator(); - while(it.hasNext()) { + while (it.hasNext()) { write(key, it.next()); } } @@ -109,7 +109,7 @@ public void progress() { protected final SerializationContext serializationContext; protected final Serializer keySerializer; protected final Serializer valSerializer; - + protected final boolean ifileReadAhead; protected final int ifileReadAheadLength; protected final int ifileBufferSize; @@ -156,7 +156,7 @@ public void progress() { // additional spills. Compressed size - so may not represent the size in the // sort buffer) protected final TezCounter additionalSpillBytesWritten; - + protected final TezCounter additionalSpillBytesRead; // Number of spills written & consumed by the same task to generate the final file protected final TezCounter numAdditionalSpills; @@ -166,21 +166,21 @@ public void progress() { final ReportPartitionStats reportPartitionStats; public ExternalSorter(OutputContext outputContext, Configuration conf, int numOutputs, - long initialMemoryAvailable) throws IOException { + long initialMemoryAvailable) throws IOException { this.outputContext = outputContext; this.conf = conf; this.localFs = (RawLocalFileSystem) FileSystem.getLocal(conf).getRaw(); this.partitions = numOutputs; reportPartitionStats = ReportPartitionStats.fromString( conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_REPORT_PARTITION_STATS, - TezRuntimeConfiguration.TEZ_RUNTIME_REPORT_PARTITION_STATS_DEFAULT)); + TezRuntimeConfiguration.TEZ_RUNTIME_REPORT_PARTITION_STATS_DEFAULT)); partitionStats = reportPartitionStats.isEnabled() ? (new long[partitions]) : null; cleanup = conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CLEANUP_FILES_ON_INTERRUPT, TezRuntimeConfiguration.TEZ_RUNTIME_CLEANUP_FILES_ON_INTERRUPT_DEFAULT); - rfs = ((LocalFileSystem)FileSystem.getLocal(this.conf)).getRaw(); + rfs = ((LocalFileSystem) FileSystem.getLocal(this.conf)).getRaw(); if (LOG.isDebugEnabled()) { LOG.debug(outputContext.getInputOutputVertexNames() + ": Initial Mem bytes : " + @@ -237,7 +237,6 @@ public ExternalSorter(OutputContext outputContext, Configuration conf, int numOu this.ifileBufferSize = conf.getInt("io.file.buffer.size", TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_BUFFER_SIZE_DEFAULT); - // Task outputs mapOutputFile = TezRuntimeUtils.instantiateTaskOutputManager(conf, outputContext); @@ -285,7 +284,7 @@ public Path getFinalOutputFile() { } protected void runCombineProcessor(TezRawKeyValueIterator kvIter, - Writer writer) throws IOException { + Writer writer) throws IOException { try { outputContext.notifyProgress(); combiner.combine(kvIter, writer); @@ -325,9 +324,9 @@ public ShuffleHeader getShuffleHeader(int reduce) { } public static long getInitialMemoryRequirement(Configuration conf, long maxAvailableTaskMemory) { - int initialMemRequestMb = + int initialMemRequestMb = conf.getInt( - TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, + TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB_DEFAULT); long reqBytes = ((long) initialMemRequestMb) << 20; //Higher bound checks are done in individual sorter implementations @@ -365,8 +364,8 @@ protected synchronized void cleanup(Path path) { try { LOG.info("Deleting " + path); rfs.delete(path, true); - } catch(IOException ioe) { - LOG.warn("Error in deleting " + path); + } catch (IOException ioe) { + LOG.warn("Error in deleting " + path); } } @@ -374,7 +373,7 @@ protected synchronized void cleanup(Map spillMap) { if (!cleanup) { return; } - for(Map.Entry entry : spillMap.entrySet()) { + for (Map.Entry entry : spillMap.entrySet()) { cleanup(entry.getValue()); } } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFile.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFile.java index 8f673185b0..d81b001ba5 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFile.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFile.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -68,8 +68,8 @@ public class IFile { public static final int RLE_MARKER = -2; // Repeat same key marker public static final int V_END_MARKER = -3; // End of values marker public static final DataInputBuffer REPEAT_KEY = new DataInputBuffer(); - static final byte[] HEADER = new byte[] { (byte) 'T', (byte) 'I', - (byte) 'F' , (byte) 0}; + static final byte[] HEADER = new byte[]{(byte) 'T', (byte) 'I', + (byte) 'F', (byte) 0}; private static final String INCOMPLETE_READ = "Requested to read %d got %d"; private static final String REQ_BUFFER_SIZE_TOO_LARGE = "Size of data %d is greater than the max allowed of %d"; @@ -121,9 +121,9 @@ public static class FileBackedInMemIFileWriter extends Writer { * @throws IOException */ public FileBackedInMemIFileWriter(Serialization keySerialization, - Serialization valSerialization, FileSystem fs, TezTaskOutput taskOutput, - Class keyClass, Class valueClass, CompressionCodec codec, TezCounter writesCounter, - TezCounter serializedBytesCounter, int cacheSize) throws IOException { + Serialization valSerialization, FileSystem fs, TezTaskOutput taskOutput, + Class keyClass, Class valueClass, CompressionCodec codec, TezCounter writesCounter, + TezCounter serializedBytesCounter, int cacheSize) throws IOException { super(keySerialization, valSerialization, new FSDataOutputStream(createBoundedBuffer(cacheSize), null), keyClass, valueClass, null, writesCounter, serializedBytesCounter); this.fs = fs; @@ -204,10 +204,9 @@ private void resetToFileBasedWriter() throws IOException { bout.reset(); } - @Override protected void writeKVPair(byte[] keyData, int keyPos, int keyLength, - byte[] valueData, int valPos, int valueLength) throws IOException { + byte[] valueData, int valPos, int valueLength) throws IOException { if (!bufferFull) { // Compute actual payload size: write RLE marker, length info and then entire data. totalSize += ((prevKey == REPEAT_KEY) ? V_END_MARKER_SIZE : 0) @@ -311,14 +310,13 @@ public static class Writer { // de-dup keys or not protected final boolean rle; - public Writer(Serialization keySerialization, Serialization valSerialization, FileSystem fs, Path file, Class keyClass, Class valueClass, CompressionCodec codec, TezCounter writesCounter, TezCounter serializedBytesCounter) throws IOException { this(keySerialization, valSerialization, fs.create(file), keyClass, valueClass, codec, - writesCounter, serializedBytesCounter); + writesCounter, serializedBytesCounter); ownOutputStream = true; } @@ -329,8 +327,8 @@ protected Writer(TezCounter writesCounter, TezCounter serializedBytesCounter, bo } public Writer(Serialization keySerialization, Serialization valSerialization, FSDataOutputStream outputStream, - Class keyClass, Class valueClass, CompressionCodec codec, TezCounter writesCounter, - TezCounter serializedBytesCounter) throws IOException { + Class keyClass, Class valueClass, CompressionCodec codec, TezCounter writesCounter, + TezCounter serializedBytesCounter) throws IOException { this(keySerialization, valSerialization, outputStream, keyClass, valueClass, codec, writesCounter, serializedBytesCounter, false); } @@ -367,14 +365,14 @@ void setupOutputStream(CompressionCodec codec) throws IOException { if (this.compressor != null) { this.compressor.reset(); this.compressedOut = CodecUtils.createOutputStream(codec, checksumOut, compressor); - this.out = new FSDataOutputStream(this.compressedOut, null); + this.out = new FSDataOutputStream(this.compressedOut, null); this.compressOutput = true; } else { LOG.warn("Could not obtain compressor from CodecPool"); - this.out = new FSDataOutputStream(checksumOut,null); + this.out = new FSDataOutputStream(checksumOut, null); } } else { - this.out = new FSDataOutputStream(checksumOut,null); + this.out = new FSDataOutputStream(checksumOut, null); } } @@ -462,7 +460,7 @@ public void append(Object key, Object value) throws IOException { if (!sameKey) { keySerializer.serialize(key); keyLength = buffer.getLength(); - assert(keyLength >= 0); + assert (keyLength >= 0); if (rle && (keyLength == previous.getLength())) { sameKey = (BufferUtils.compare(previous, buffer) == 0); } @@ -471,7 +469,7 @@ public void append(Object key, Object value) throws IOException { // Append the 'value' valueSerializer.serialize(value); int valueLength = buffer.getLength() - keyLength; - assert(valueLength >= 0); + assert (valueLength >= 0); if (!sameKey) { //dump entire key value pair writeKVPair(buffer.getData(), 0, keyLength, buffer.getData(), @@ -517,7 +515,7 @@ public void appendValue(Object value) throws IOException { */ public void appendValue(DataInputBuffer value) throws IOException { int valueLength = value.getLength() - value.getPosition(); - assert(valueLength >= 0); + assert (valueLength >= 0); writeValue(value.getData(), value.getPosition(), valueLength); buffer.reset(); ++numRecordsWritten; @@ -532,7 +530,7 @@ public void appendValue(DataInputBuffer value) throws IOException { * @throws IOException */ public void appendValues(Iterator valuesItr) throws IOException { - while(valuesItr.hasNext()) { + while (valuesItr.hasNext()) { appendValue(valuesItr.next()); } } @@ -551,7 +549,7 @@ public void appendKeyValues(K key, Iterator valuesItr) throws IOExcept append(key, valuesItr.next()); //append first KV pair } //append the remaining values - while(valuesItr.hasNext()) { + while (valuesItr.hasNext()) { appendValue(valuesItr.next()); } } @@ -569,10 +567,10 @@ public void appendKeyValues(K key, Iterator valuesItr) throws IOExcept */ public void append(DataInputBuffer key, DataInputBuffer value) throws IOException { int keyLength = key.getLength() - key.getPosition(); - assert(key == REPEAT_KEY || keyLength >=0); + assert (key == REPEAT_KEY || keyLength >= 0); int valueLength = value.getLength() - value.getPosition(); - assert(valueLength >= 0); + assert (valueLength >= 0); sameKey = (key == REPEAT_KEY); if (!sameKey && rle) { @@ -606,7 +604,7 @@ protected void writeValue(byte[] data, int offset, int length) throws IOExceptio } protected void writeKVPair(byte[] keyData, int keyPos, int keyLength, - byte[] valueData, int valPos, int valueLength) throws IOException { + byte[] valueData, int valPos, int valueLength) throws IOException { writeValueMarker(out); WritableUtils.writeVInt(out, keyLength); WritableUtils.writeVInt(out, valueLength); @@ -650,7 +648,7 @@ protected void writeValueMarker(DataOutputStream out) throws IOException { } // Required for mark/reset - public DataOutputStream getOutputStream () { + public DataOutputStream getOutputStream() { return out; } @@ -678,14 +676,14 @@ public static class Reader { public enum KeyState {NO_KEY, NEW_KEY, SAME_KEY} - private static final int DEFAULT_BUFFER_SIZE = 128*1024; + private static final int DEFAULT_BUFFER_SIZE = 128 * 1024; @VisibleForTesting // Not final for testing protected static int MAX_BUFFER_SIZE - = Integer.MAX_VALUE - 8; // The maximum array size is a little less than the - // max integer value. Trying to create a larger array - // will result in an OOM exception. The exact value - // is JVM dependent so setting it to max int - 8 to be safe. + = Integer.MAX_VALUE - 8; // The maximum array size is a little less than the + // max integer value. Trying to create a larger array + // will result in an OOM exception. The exact value + // is JVM dependent so setting it to max int - 8 to be safe. // Count records read from disk private long numRecordsRead = 0; @@ -742,10 +740,10 @@ public Reader(FileSystem fs, Path file, * @throws IOException */ public Reader(InputStream in, long length, - CompressionCodec codec, - TezCounter readsCounter, TezCounter bytesReadCounter, - boolean readAhead, int readAheadLength, - int bufferSize) throws IOException { + CompressionCodec codec, + TezCounter readsCounter, TezCounter bytesReadCounter, + boolean readAhead, int readAheadLength, + int bufferSize) throws IOException { this(in, ((in != null) ? (length - HEADER.length) : length), codec, readsCounter, bytesReadCounter, readAhead, readAheadLength, bufferSize, ((in != null) ? isCompressedFlagEnabled(in) : false)); @@ -809,7 +807,7 @@ public Reader(InputStream in, long length, * @throws IOException */ public static void readToMemory(byte[] buffer, InputStream in, int compressedLength, - CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLength) + CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLength) throws IOException { boolean isCompressed = IFile.Reader.isCompressedFlagEnabled(in); IFileInputStream checksumIn = new IFileInputStream(in, @@ -840,10 +838,10 @@ public static void readToMemory(byte[] buffer, InputStream in, int compressedLen throw new IOException("Unexpected extra bytes from input stream"); } } catch (IOException ioe) { - if(in != null) { + if (in != null) { try { in.close(); - } catch(IOException e) { + } catch (IOException e) { LOG.debug("Exception in closing {}", in, e); } } @@ -866,7 +864,7 @@ public static void readToMemory(byte[] buffer, InputStream in, int compressedLen * @throws IOException */ public static long readToDisk(OutputStream out, InputStream in, long length, - boolean ifileReadAhead, int ifileReadAheadLength) + boolean ifileReadAhead, int ifileReadAheadLength) throws IOException { final int BYTES_TO_READ = 64 * 1024; byte[] buf = new byte[BYTES_TO_READ]; @@ -974,11 +972,11 @@ protected boolean positionToNextRecord(DataInput dIn) throws IOException { // Sanity check if (currentKeyLength != RLE_MARKER && currentKeyLength < 0) { throw new IOException("Rec# " + recNo + ": Negative key-length: " + - currentKeyLength + " PreviousKeyLen: " + prevKeyLength); + currentKeyLength + " PreviousKeyLen: " + prevKeyLength); } if (currentValueLength < 0) { throw new IOException("Rec# " + recNo + ": Negative value-length: " + - currentValueLength); + currentValueLength); } return true; } @@ -990,7 +988,7 @@ public final boolean nextRawKey(DataInputBuffer key) throws IOException { private static byte[] createLargerArray(int currentLength) { if (currentLength > MAX_BUFFER_SIZE) { throw new IllegalArgumentException( - String.format(REQ_BUFFER_SIZE_TOO_LARGE, currentLength, MAX_BUFFER_SIZE)); + String.format(REQ_BUFFER_SIZE_TOO_LARGE, currentLength, MAX_BUFFER_SIZE)); } int newLength; if (currentLength > (MAX_BUFFER_SIZE - currentLength)) { @@ -1012,7 +1010,7 @@ public KeyState readRawKey(DataInputBuffer key) throws IOException { } return KeyState.NO_KEY; } - if(currentKeyLength == RLE_MARKER) { + if (currentKeyLength == RLE_MARKER) { // get key length from original key key.reset(keyBytes, originalKeyLength); return KeyState.SAME_KEY; @@ -1095,5 +1093,4 @@ public void disableChecksumValidation() { checksumIn.disableChecksumValidation(); } } - } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFileInputStream.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFileInputStream.java index c5853d4a46..a9acd255e1 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFileInputStream.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFileInputStream.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -35,14 +35,15 @@ import org.apache.hadoop.io.ReadaheadPool; import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest; import org.apache.hadoop.util.DataChecksum; + /** * A checksum input stream, used for IFiles. * Used to validate the checksum of files created by {@link IFileOutputStream}. -*/ + */ @InterfaceAudience.Private @InterfaceStability.Unstable public class IFileInputStream extends InputStream { - + private final InputStream in; //The input stream to be verified for checksum. private final FileDescriptor inFd; // the file descriptor, if it is known private final long length; //The total length of the input file @@ -105,9 +106,9 @@ private static FileDescriptor getFileDescriptorIfAvail(InputStream in) { FileDescriptor fd = null; try { if (in instanceof HasFileDescriptor) { - fd = ((HasFileDescriptor)in).getFileDescriptor(); + fd = ((HasFileDescriptor) in).getFileDescriptor(); } else if (in instanceof FileInputStream) { - fd = ((FileInputStream)in).getFD(); + fd = ((FileInputStream) in).getFD(); } } catch (IOException e) { LOG.info("Unable to determine FileDescriptor", e); @@ -127,7 +128,7 @@ public void close() throws IOException { } if (currentOffset < dataLength && !disableChecksumValidation) { byte[] t = new byte[Math.min((int) - (Integer.MAX_VALUE & (dataLength - currentOffset)), 32 * 1024)]; + (Integer.MAX_VALUE & (dataLength - currentOffset)), 32 * 1024)]; while (currentOffset < dataLength) { int n = read(t, 0, t.length); if (0 == n) { @@ -137,29 +138,29 @@ public void close() throws IOException { } in.close(); } - + @Override public long skip(long n) throws IOException { - throw new IOException("Skip not supported for IFileInputStream"); + throw new IOException("Skip not supported for IFileInputStream"); } - + public long getPosition() { return (currentOffset >= dataLength) ? dataLength : currentOffset; } - + public long getSize() { return checksumSize; } private void checksum(byte[] b, int off, int len) { - if(len >= buffer.length) { + if (len >= buffer.length) { sum.update(buffer, 0, offset); offset = 0; sum.update(b, off, len); return; } final int remaining = buffer.length - offset; - if(len > remaining) { + if (len > remaining) { sum.update(buffer, 0, offset); offset = 0; } @@ -167,7 +168,7 @@ private void checksum(byte[] b, int off, int len) { System.arraycopy(b, off, buffer, offset, len); offset += len; } - + /** * Read bytes from the stream. * At EOF, checksum is validated, but the checksum @@ -182,7 +183,7 @@ public int read(byte[] b, int off, int len) throws IOException { doReadahead(); - return doRead(b,off,len); + return doRead(b, off, len); } private void doReadahead() { @@ -204,8 +205,7 @@ public int readWithChecksum(byte[] b, int off, int len) throws IOException { if (currentOffset == length) { return -1; - } - else if (currentOffset >= dataLength) { + } else if (currentOffset >= dataLength) { // If the previous read drained off all the data, then just return // the checksum now. Note that checksum validation would have // happened in the earlier read @@ -213,13 +213,13 @@ else if (currentOffset >= dataLength) { if (len < lenToCopy) { lenToCopy = len; } - System.arraycopy(csum, (int) (currentOffset - dataLength), b, off, + System.arraycopy(csum, (int) (currentOffset - dataLength), b, off, lenToCopy); currentOffset += lenToCopy; return lenToCopy; } - int bytesRead = doRead(b,off,len); + int bytesRead = doRead(b, off, len); if (currentOffset == dataLength) { if (len >= bytesRead + checksumSize) { @@ -231,22 +231,22 @@ else if (currentOffset >= dataLength) { return bytesRead; } - private int doRead(byte[]b, int off, int len) throws IOException { - + private int doRead(byte[] b, int off, int len) throws IOException { + // If we are trying to read past the end of data, just read // the left over data int origLen = len; if (currentOffset + len > dataLength) { len = (int) (dataLength - currentOffset); } - + int bytesRead = in.read(b, off, len); if (bytesRead < 0) { String mesg = " CurrentOffset=" + currentOffset + ", offset=" + offset + ", off=" + off + - ", dataLength=" + dataLength + + ", dataLength=" + dataLength + ", origLen=" + origLen + ", len=" + len + ", length=" + length + @@ -262,7 +262,7 @@ private int doRead(byte[]b, int off, int len) throws IOException { if (disableChecksumValidation) { return bytesRead; } - + if (currentOffset == dataLength) { //TODO: add checksumSize to currentOffset. // The last four bytes are checksum. Strip them and verify @@ -272,13 +272,13 @@ private int doRead(byte[]b, int off, int len) throws IOException { if (!sum.compare(csum, 0)) { String mesg = "CurrentOffset=" + currentOffset + ", off=" + offset + - ", dataLength=" + dataLength + + ", dataLength=" + dataLength + ", origLen=" + origLen + ", len=" + len + ", length=" + length + - ", checksumSize=" + checksumSize+ + ", checksumSize=" + checksumSize + ", csum=" + Arrays.toString(csum) + - ", sum=" + sum; + ", sum=" + sum; LOG.info(mesg); throw new ChecksumException("Checksum Error: " + mesg, 0); @@ -287,13 +287,12 @@ private int doRead(byte[]b, int off, int len) throws IOException { return bytesRead; } - @Override - public int read() throws IOException { + public int read() throws IOException { b[0] = 0; - int l = read(b,0,1); - if (l < 0) return l; - + int l = read(b, 0, 1); + if (l < 0) return l; + // Upgrade the b[0] to an int so as not to misinterpret the // first bit of the byte as a sign bit int result = 0xFF & b[0]; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFileOutputStream.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFileOutputStream.java index 5ec0537288..bf1a551c8f 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFileOutputStream.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/IFileOutputStream.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,6 +25,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.util.DataChecksum; + /** * A Checksum output stream. * Checksum for the contents of the file is calculated and @@ -85,19 +86,19 @@ public void finish() throws IOException { finished = true; sum.update(buffer, 0, offset); sum.writeValue(barray, 0, false); - out.write (barray, 0, sum.getChecksumSize()); + out.write(barray, 0, sum.getChecksumSize()); out.flush(); } private void checksum(byte[] b, int off, int len) { - if(len >= buffer.length) { + if (len >= buffer.length) { sum.update(buffer, 0, offset); offset = 0; sum.update(b, off, len); return; } final int remaining = buffer.length - offset; - if(len > remaining) { + if (len > remaining) { sum.update(buffer, 0, offset); offset = 0; } @@ -121,13 +122,12 @@ private void checksum(byte[] b, int off, int len) { @Override public void write(byte[] b, int off, int len) throws IOException { checksum(b, off, len); - out.write(b,off,len); + out.write(b, off, len); } @Override public void write(int b) throws IOException { barray[0] = (byte) (b & 0xFF); - write(barray,0,1); + write(barray, 0, 1); } - } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java index 08786c9b2c..e666ed3026 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java @@ -1,20 +1,20 @@ - /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.runtime.library.common.sort.impl; import java.io.IOException; @@ -73,9 +73,9 @@ @SuppressWarnings({"unchecked", "rawtypes"}) public class PipelinedSorter extends ExternalSorter { - + private static final Logger LOG = LoggerFactory.getLogger(PipelinedSorter.class); - + /** * The size of each record in the index file for the map-outputs. */ @@ -84,7 +84,7 @@ public class PipelinedSorter extends ExternalSorter { private final static int APPROX_HEADER_LENGTH = 150; private final int partitionBits; - + private static final int PARTITION = 0; // partition offset in acct private static final int KEYSTART = 1; // key offset in acct private static final int VALSTART = 2; // val offset in acct @@ -104,11 +104,11 @@ public class PipelinedSorter extends ExternalSorter { private int bufferOverflowRecursion; // Merger - private final SpanMerger merger; + private final SpanMerger merger; private final ExecutorService sortmaster; private final ArrayList indexCacheList = - new ArrayList(); + new ArrayList(); private final boolean pipelinedShuffle; @@ -133,7 +133,7 @@ public class PipelinedSorter extends ExternalSorter { // TODO Set additional countesr - total bytes written, spills etc. public PipelinedSorter(OutputContext outputContext, Configuration conf, int numOutputs, - long initialMemoryAvailable) throws IOException { + long initialMemoryAvailable) throws IOException { super(outputContext, conf, numOutputs, initialMemoryAvailable); lazyAllocateMem = this.conf.getBoolean(TezRuntimeConfiguration @@ -162,7 +162,7 @@ public PipelinedSorter(OutputContext outputContext, Configuration conf, int numO StringBuilder initialSetupLogLine = new StringBuilder("Setting up PipelinedSorter for ") .append(outputContext.getInputOutputVertexNames()).append(": "); - partitionBits = bitcount(partitions)+1; + partitionBits = bitcount(partitions) + 1; boolean confPipelinedShuffle = this.conf.getBoolean(TezRuntimeConfiguration .TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED, TezRuntimeConfiguration @@ -179,8 +179,8 @@ public PipelinedSorter(OutputContext outputContext, Configuration conf, int numO initialSetupLogLine.append(", UsingHashComparator="); // k/v serialization - if(comparator instanceof ProxyComparator) { - hasher = (ProxyComparator)comparator; + if (comparator instanceof ProxyComparator) { + hasher = (ProxyComparator) comparator; initialSetupLogLine.append(true); } else { hasher = null; @@ -192,7 +192,7 @@ public PipelinedSorter(OutputContext outputContext, Configuration conf, int numO long totalCapacityWithoutMeta = 0; long availableMem = maxMemLimit; int numBlocks = 0; - while(availableMem > 0) { + while (availableMem > 0) { long size = Math.min(availableMem, computeBlockSize(availableMem, maxMemLimit)); int sizeWithoutMeta = (int) ((size) - (size % METASIZE)); totalCapacityWithoutMeta += sizeWithoutMeta; @@ -208,7 +208,7 @@ public PipelinedSorter(OutputContext outputContext, Configuration conf, int numO allocateSpace(); //Allocate the first block if (!lazyAllocateMem) { LOG.info("Pre allocating rest of memory buffers upfront"); - while(allocateSpace() != null); + while (allocateSpace() != null) ; } initialSetupLogLine.append("#blocks=").append(maxNumberOfBlocks); @@ -229,10 +229,10 @@ public PipelinedSorter(OutputContext outputContext, Configuration conf, int numO span = new SortSpan(buffers.get(bufferIndex), 1024 * 1024, 16, this.comparator); merger = new SpanMerger(); // SpanIterators are comparable - final int sortThreads = - this.conf.getInt( - TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SORTER_SORT_THREADS, - TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SORTER_SORT_THREADS_DEFAULT); + final int sortThreads = + this.conf.getInt( + TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SORTER_SORT_THREADS, + TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SORTER_SORT_THREADS_DEFAULT); sortmaster = Executors.newFixedThreadPool(sortThreads, new ThreadFactoryBuilder().setDaemon(true) .setNameFormat("Sorter {" + TezUtilsInternal.cleanVertexName(outputContext.getTaskVertexName()) + " -> " @@ -274,7 +274,6 @@ ByteBuffer allocateSpace() { return space; } - @VisibleForTesting int computeBlockSize(long availableMem, long maxAllocatedMemory) { int maxBlockSize = 0; @@ -316,17 +315,17 @@ int computeBlockSize(long availableMem, long maxAllocatedMemory) { private int bitcount(int n) { int bit = 0; - while(n!=0) { + while (n != 0) { bit++; n >>= 1; } return bit; } - + public void sort() throws IOException { SortSpan newSpan = span.next(); - if(newSpan == null) { + if (newSpan == null) { //avoid sort/spill of empty span StopWatch stopWatch = new StopWatch(); stopWatch.start(); @@ -344,20 +343,20 @@ public void sort() throws IOException { // Use the next buffer bufferIndex = (bufferIndex + 1) % buffers.size(); bufferUsage.set(bufferIndex, bufferUsage.get(bufferIndex) + 1); - int items = 1024*1024; + int items = 1024 * 1024; int perItem = 16; - if(span.length() != 0) { + if (span.length() != 0) { items = span.length(); - perItem = span.kvbuffer.limit()/items; - items = (int) ((span.capacity)/(METASIZE+perItem)); - if(items > 1024*1024) { - // our goal is to have 1M splits and sort early - items = 1024*1024; + perItem = span.kvbuffer.limit() / items; + items = (int) ((span.capacity) / (METASIZE + perItem)); + if (items > 1024 * 1024) { + // our goal is to have 1M splits and sort early + items = 1024 * 1024; } } Preconditions.checkArgument(buffers.get(bufferIndex) != null, "block should not be empty"); //TODO: fix per item being passed. - span = new SortSpan((ByteBuffer)buffers.get(bufferIndex).clear(), (1024*1024), + span = new SortSpan((ByteBuffer) buffers.get(bufferIndex).clear(), (1024 * 1024), perItem, ConfigUtils.getIntermediateOutputKeyComparator(this.conf)); } else { // queue up the sort @@ -372,9 +371,9 @@ public void sort() throws IOException { } // if pipelined shuffle is enabled, this method is called to send events for every spill - private void sendPipelinedShuffleEvents() throws IOException{ + private void sendPipelinedShuffleEvents() throws IOException { List events = Lists.newLinkedList(); - String pathComponent = (outputContext.getUniqueIdentifier() + "_" + (numSpills-1)); + String pathComponent = (outputContext.getUniqueIdentifier() + "_" + (numSpills - 1)); ShuffleUtils.generateEventOnSpill(events, isFinalMergeEnabled(), false, outputContext, (numSpills - 1), indexCacheList.get(numSpills - 1), partitions, sendEmptyPartitionDetails, pathComponent, partitionStats, @@ -397,16 +396,16 @@ public void write(Object key, Object value) * storage to store one METADATA. */ synchronized void collect(Object key, Object value, final int partition - ) throws IOException { + ) throws IOException { if (key.getClass() != serializationContext.getKeyClass()) { throw new IOException("Type mismatch in key from map: expected " - + serializationContext.getKeyClass().getName() + ", received " - + key.getClass().getName()); + + serializationContext.getKeyClass().getName() + ", received " + + key.getClass().getName()); } if (value.getClass() != serializationContext.getValueClass()) { throw new IOException("Type mismatch in value from map: expected " - + serializationContext.getValueClass().getName() + ", received " - + value.getClass().getName()); + + serializationContext.getValueClass().getName() + ", received " + + value.getClass().getName()); } if (partition < 0 || partition >= partitions) { throw new IOException("Illegal partition for " + key + " (" + @@ -425,10 +424,10 @@ synchronized void collect(Object key, Object value, final int partition int valend = -1; try { keySerializer.serialize(key); - valstart = span.kvbuffer.position(); + valstart = span.kvbuffer.position(); valSerializer.serialize(value); valend = span.kvbuffer.position(); - } catch(BufferOverflowException overflow) { + } catch (BufferOverflowException overflow) { // restore limit span.kvbuffer.position(keystart); this.sort(); @@ -450,7 +449,7 @@ synchronized void collect(Object key, Object value, final int partition int prefix = 0; - if(hasher != null) { + if (hasher != null) { prefix = hasher.getProxy(key); } @@ -484,7 +483,7 @@ private void adjustSpillCounters(long rawLength, long compLength) { // it is guaranteed that when spillSingleRecord is called, there is // no merger spans queued in executor. private void spillSingleRecord(final Object key, final Object value, - int partition) throws IOException { + int partition) throws IOException { final TezSpillRecord spillRec = new TezSpillRecord(partitions); // getSpillFileForWrite with size -1 as the serialized size of KV pair is still unknown final Path filename = mapOutputFile.getSpillFileForWrite(numSpills, -1); @@ -544,15 +543,15 @@ private void spillSingleRecord(final Object key, final Object value, indexCacheList.add(spillRec); ++numSpills; if (!isFinalMergeEnabled()) { - fileOutputByteCounter.increment(rfs.getFileStatus(filename).getLen()); - //No final merge. Set the number of files offered via shuffle-handler - numShuffleChunks.setValue(numSpills); + fileOutputByteCounter.increment(rfs.getFileStatus(filename).getLen()); + //No final merge. Set the number of files offered via shuffle-handler + numShuffleChunks.setValue(numSpills); } if (pipelinedShuffle) { sendPipelinedShuffleEvents(); } } finally { - out.close(); + out.close(); } } @@ -563,7 +562,7 @@ public boolean spill(boolean ignoreEmptySpills) throws IOException { boolean ret = merger.ready(); // if merger returned false and ignore merge is true, // then return directly without spilling - if (!ret && ignoreEmptySpills){ + if (!ret && ignoreEmptySpills) { return false; } } catch (InterruptedException e) { @@ -575,10 +574,10 @@ public boolean spill(boolean ignoreEmptySpills) throws IOException { // create spill file final long size = capacity + - + (partitions * APPROX_HEADER_LENGTH); + +(partitions * APPROX_HEADER_LENGTH); final TezSpillRecord spillRec = new TezSpillRecord(partitions); final Path filename = - mapOutputFile.getSpillFileForWrite(numSpills, size); + mapOutputFile.getSpillFileForWrite(numSpills, size); spillFilePaths.put(numSpills, filename); out = rfs.create(filename, true, 4096); ensureSpillFilePermissions(filename, rfs); @@ -603,7 +602,7 @@ public boolean spill(boolean ignoreEmptySpills) throws IOException { while (kvIter.next()) { writer.append(kvIter.getKey(), kvIter.getValue()); } - } else { + } else { if (hasNext) { runCombineProcessor(kvIter, writer); } @@ -627,8 +626,8 @@ public boolean spill(boolean ignoreEmptySpills) throws IOException { } Path indexFilename = - mapOutputFile.getSpillIndexFileForWrite(numSpills, partitions - * MAP_OUTPUT_INDEX_RECORD_LENGTH); + mapOutputFile.getSpillIndexFileForWrite(numSpills, partitions + * MAP_OUTPUT_INDEX_RECORD_LENGTH); spillFileIndexPaths.put(numSpills, indexFilename); spillRec.writeToFile(indexFilename, conf, localFs); //TODO: honor cache limits @@ -693,8 +692,7 @@ public void flush() throws IOException { //safe to clean up buffers.clear(); - - if(indexCacheList.isEmpty()) { + if (indexCacheList.isEmpty()) { /* * If we do not have this check, and if the task gets killed in the middle, it can throw * NPE leading to distraction when debugging. @@ -764,7 +762,7 @@ public void flush() throws IOException { if (LOG.isDebugEnabled()) { LOG.debug(outputContext.getInputOutputVertexNames() + ": " + "numSpills: " + numSpills + ", finalOutputFile:" + finalOutputFile + ", finalIndexFile:" - + finalIndexFile); + + finalIndexFile); } //The output stream for the final single output file FSDataOutputStream finalOut = rfs.create(finalOutputFile, true, 4096); @@ -850,7 +848,7 @@ public void flush() throws IOException { spillFileIndexPaths.clear(); spillFilePaths.clear(); - } catch(InterruptedException ie) { + } catch (InterruptedException ie) { if (cleanup) { cleanup(); } @@ -869,32 +867,35 @@ public final List close() throws IOException { return finalEvents; } - private interface PartitionedRawKeyValueIterator extends TezRawKeyValueIterator { int getPartition(); + Integer peekPartition(); } - private static class BufferStreamWrapper extends OutputStream - { + private static class BufferStreamWrapper extends OutputStream { private final ByteBuffer out; + public BufferStreamWrapper(ByteBuffer out) { this.out = out; } - + @Override - public void write(int b) throws IOException { out.put((byte)b); } + public void write(int b) throws IOException {out.put((byte) b);} + @Override - public void write(byte[] b) throws IOException { out.put(b); } + public void write(byte[] b) throws IOException {out.put(b);} + @Override - public void write(byte[] b, int off, int len) throws IOException { out.put(b, off, len); } + public void write(byte[] b, int off, int len) throws IOException {out.put(b, off, len);} } private static final class InputByteBuffer extends DataInputBuffer { - private byte[] buffer = new byte[256]; + private byte[] buffer = new byte[256]; private ByteBuffer wrapped = ByteBuffer.wrap(buffer); + private void resize(int length) { - if(length > buffer.length || (buffer.length > 10 * (1+length))) { + if (length > buffer.length || (buffer.length > 10 * (1 + length))) { // scale down as well as scale up across values buffer = new byte[length]; wrapped = ByteBuffer.wrap(buffer); @@ -936,14 +937,13 @@ private final class SortSpan implements IndexedSortable { private boolean reinit = false; private int capacity; - public SortSpan(ByteBuffer source, int maxItems, int perItem, RawComparator comparator) { capacity = source.remaining(); - int metasize = METASIZE*maxItems; + int metasize = METASIZE * maxItems; int dataSize = maxItems * perItem; - if(capacity < (metasize+dataSize)) { + if (capacity < (metasize + dataSize)) { // try to allocate less meta space, because we have sample data - metasize = METASIZE*(capacity/(perItem+METASIZE)); + metasize = METASIZE * (capacity / (perItem + METASIZE)); } ByteBuffer reserved = source.duplicate(); reserved.mark(); @@ -957,21 +957,21 @@ public SortSpan(ByteBuffer source, int maxItems, int perItem, RawComparator comp rawkvmeta = kvmetabuffer.array(); kvmetabase = kvmetabuffer.arrayOffset(); kvmeta = kvmetabuffer - .order(ByteOrder.nativeOrder()) - .asIntBuffer(); + .order(ByteOrder.nativeOrder()) + .asIntBuffer(); out = new NonSyncDataOutputStream( - new BufferStreamWrapper(kvbuffer)); + new BufferStreamWrapper(kvbuffer)); this.comparator = comparator; } public SpanIterator sort(IndexedSorter sorter) { long start = System.currentTimeMillis(); - if(length() > 1) { + if (length() > 1) { sorter.sort(this, 0, length(), progressable); } LOG.info(outputContext.getInputOutputVertexNames() + ": " + "done sorting span=" + index + ", length=" + length() + ", " + "time=" + (System.currentTimeMillis() - start)); - return new SpanIterator((SortSpan)this); + return new SpanIterator((SortSpan) this); } int offsetFor(int i) { @@ -992,8 +992,8 @@ public void swap(final int mi, final int mj) { protected int compareKeys(final int kvi, final int kvj) { final int istart = kvmeta.get(kvi + KEYSTART); final int jstart = kvmeta.get(kvj + KEYSTART); - final int ilen = kvmeta.get(kvi + VALSTART) - istart; - final int jlen = kvmeta.get(kvj + VALSTART) - jstart; + final int ilen = kvmeta.get(kvi + VALSTART) - istart; + final int jlen = kvmeta.get(kvj + VALSTART) - jstart; if (ilen == 0 || jlen == 0) { if (ilen == jlen) { @@ -1007,11 +1007,10 @@ protected int compareKeys(final int kvi, final int kvj) { // sort by key final int cmp = comparator.compare(buf, off + istart, ilen, buf, off + jstart, jlen); - if(cmp == 0) eq++; + if (cmp == 0) eq++; return cmp; } - public int compare(final int mi, final int mj) { final int kvi = offsetFor(mi); final int kvj = offsetFor(mj); @@ -1026,13 +1025,13 @@ public int compare(final int mi, final int mj) { public SortSpan next() { ByteBuffer remaining = end(); - if(remaining != null) { + if (remaining != null) { SortSpan newSpan = null; int items = length(); - int perItem = kvbuffer.position()/items; + int perItem = kvbuffer.position() / items; if (reinit) { //next mem block //quite possible that the previous span had a length of 1. It is better to reinit here for new span. - items = 1024*1024; + items = 1024 * 1024; perItem = 16; } final RawComparator newComparator = ConfigUtils.getIntermediateOutputKeyComparator(conf); @@ -1044,7 +1043,7 @@ public SortSpan next() { System.identityHashCode(newComparator)); } newSpan = new SortSpan(remaining, items, perItem, newComparator); - newSpan.index = index+1; + newSpan.index = index + 1; LOG.info( String.format(outputContext.getInputOutputVertexNames() + ": " + "New Span%d.length = %d, perItem = %d", newSpan.index, newSpan.length(), perItem) + ", counter:" + mapOutputRecordCounter.getValue()); @@ -1054,7 +1053,7 @@ public SortSpan next() { } public int length() { - return kvmeta.limit()/NMETA; + return kvmeta.limit() / NMETA; } public ByteBuffer end() { @@ -1064,13 +1063,13 @@ public ByteBuffer end() { kvbuffer.limit(kvbuffer.position()); kvmeta.limit(kvmeta.position()); int items = length(); - if(items == 0) { + if (items == 0) { return null; } - int perItem = kvbuffer.position()/items; + int perItem = kvbuffer.position() / items; LOG.info(outputContext.getInputOutputVertexNames() + ": " + String.format("Span%d.length = %d, perItem = %d", index, length(), perItem)); - if(remaining.remaining() < METASIZE+perItem) { + if (remaining.remaining() < METASIZE + perItem) { //Check if we can get the next Buffer from the main buffer list ByteBuffer space = allocateSpace(); if (space != null) { @@ -1090,28 +1089,28 @@ public int compareInternal(final DataInputBuffer needle, final int needlePart, f final int valstart; final int partition; partition = kvmeta.get(this.offsetFor(index) + PARTITION); - if(partition != needlePart) { - cmp = (partition-needlePart); + if (partition != needlePart) { + cmp = (partition - needlePart); } else { keystart = kvmeta.get(this.offsetFor(index) + KEYSTART); valstart = kvmeta.get(this.offsetFor(index) + VALSTART); final byte[] buf = kvbuffer.array(); final int off = kvbuffer.arrayOffset(); cmp = comparator.compare(buf, - keystart + off , (valstart - keystart), + keystart + off, (valstart - keystart), needle.getData(), needle.getPosition(), (needle.getLength() - needle.getPosition())); } return cmp; } - + public long getEq() { return eq; } - + @Override public String toString() { - return String.format("Span[%d,%d]", NMETA*kvmeta.capacity(), kvbuffer.limit()); + return String.format("Span[%d,%d]", NMETA * kvmeta.capacity(), kvbuffer.limit()); } } @@ -1131,10 +1130,10 @@ public SpanIterator(SortSpan span) { this.kvmeta = span.kvmeta; this.kvbuffer = span.kvbuffer; this.span = span; - this.maxindex = (kvmeta.limit()/NMETA) - 1; + this.maxindex = (kvmeta.limit() / NMETA) - 1; } - public DataInputBuffer getKey() { + public DataInputBuffer getKey() { final int keystart = kvmeta.get(span.offsetFor(kvindex) + KEYSTART); final int valstart = kvmeta.get(span.offsetFor(kvindex) + VALSTART); final byte[] buf = kvbuffer.array(); @@ -1154,9 +1153,9 @@ public DataInputBuffer getValue() { public boolean next() { // caveat: since we use this as a comparable in the merger - if(kvindex == maxindex) return false; + if (kvindex == maxindex) return false; kvindex += 1; - if(kvindex % 100 == 0) { + if (kvindex % 100 == 0) { progress.set(1 - ((maxindex - kvindex) / (float) maxindex)); } return true; @@ -1170,7 +1169,7 @@ public boolean hasNext() { public void close() { } - public Progress getProgress() { + public Progress getProgress() { return progress; } @@ -1188,7 +1187,7 @@ public Integer peekPartition() { if (!hasNext()) { return null; } else { - return kvmeta.get(span.offsetFor(kvindex + 1) + PARTITION); + return kvmeta.get(span.offsetFor(kvindex + 1) + PARTITION); } } @@ -1210,52 +1209,52 @@ public String toString() { * bisect returns the next insertion point for a given raw key, skipping keys * which are <= needle using a binary search instead of a linear comparison. * This is massively efficient when long strings of identical keys occur. - * @param needle + * @param needle * @param needlePart * @return */ int bisect(DataInputBuffer needle, int needlePart) { int start = kvindex; - int end = maxindex-1; + int end = maxindex - 1; int mid = start; int cmp = 0; - if(end - start < minrun) { + if (end - start < minrun) { return 0; } - if(span.compareInternal(needle, needlePart, start) > 0) { + if (span.compareInternal(needle, needlePart, start) > 0) { return kvindex; } - + // bail out early if we haven't got a min run - if(span.compareInternal(needle, needlePart, start+minrun) > 0) { + if (span.compareInternal(needle, needlePart, start + minrun) > 0) { return 0; } - if(span.compareInternal(needle, needlePart, end) < 0) { + if (span.compareInternal(needle, needlePart, end) < 0) { return end - kvindex; } - + boolean found = false; - + // we sort 100k items, the max it can do is 20 loops, but break early - for(int i = 0; start < end && i < 16; i++) { - mid = start + (end - start)/2; + for (int i = 0; start < end && i < 16; i++) { + mid = start + (end - start) / 2; cmp = span.compareInternal(needle, needlePart, mid); - if(cmp == 0) { + if (cmp == 0) { start = mid; found = true; - } else if(cmp < 0) { - start = mid; + } else if (cmp < 0) { + start = mid; found = true; } - if(cmp > 0) { + if (cmp > 0) { end = mid; } } - if(found) { + if (found) { return start - kvindex; } return 0; @@ -1267,8 +1266,8 @@ private static class SortTask extends CallableWithNdc { private final IndexedSorter sorter; public SortTask(SortSpan sortable, IndexedSorter sorter) { - this.sortable = sortable; - this.sorter = sorter; + this.sortable = sortable; + this.sorter = sorter; } @Override @@ -1281,12 +1280,17 @@ private class PartitionFilter implements TezRawKeyValueIterator { private final PartitionedRawKeyValueIterator iter; private int partition; private boolean dirty = false; + public PartitionFilter(PartitionedRawKeyValueIterator iter) { this.iter = iter; } - public DataInputBuffer getKey() throws IOException { return iter.getKey(); } - public DataInputBuffer getValue() throws IOException { return iter.getValue(); } - public void close() throws IOException { } + + public DataInputBuffer getKey() throws IOException {return iter.getKey();} + + public DataInputBuffer getValue() throws IOException {return iter.getValue();} + + public void close() throws IOException {} + public Progress getProgress() { return new Progress(); } @@ -1297,13 +1301,13 @@ public boolean isSameKey() throws IOException { } public boolean next() throws IOException { - if(dirty || iter.next()) { + if (dirty || iter.next()) { int prefix = iter.getPartition(); - if((prefix >>> (32 - partitionBits)) == partition) { + if ((prefix >>> (32 - partitionBits)) == partition) { dirty = false; // we found what we were looking for, good return true; - } else if(!dirty) { + } else if (!dirty) { dirty = true; // we did a lookahead and failed to find partition } } @@ -1343,6 +1347,7 @@ private static class SpanHeap extends java.util.PriorityQueue { public SpanHeap() { super(256); } + /** * {@link PriorityQueue}.poll() by a different name * @return @@ -1363,7 +1368,7 @@ private final class SpanMerger implements PartitionedRawKeyValueIterator { InputByteBuffer value = new InputByteBuffer(); int partition; - private ArrayList< Future> futures = new ArrayList< Future>(); + private ArrayList> futures = new ArrayList>(); private SpanHeap heap = new SpanHeap(); private PartitionFilter partIter; @@ -1372,14 +1377,14 @@ private final class SpanMerger implements PartitionedRawKeyValueIterator { private SpanIterator horse; private long total = 0; private long eq = 0; - + public SpanMerger() { // SpanIterators are comparable partIter = new PartitionFilter(this); } public final void add(SpanIterator iter) { - if(iter.next()) { + if (iter.next()) { heap.add(iter); } } @@ -1392,7 +1397,7 @@ public final boolean ready() throws IOException, InterruptedException { int numSpanItr = futures.size(); try { SpanIterator iter = null; - while(this.futures.size() > 0) { + while (this.futures.size() > 0) { Future futureIter = this.futures.remove(0); iter = futureIter.get(); this.add(iter); @@ -1402,15 +1407,15 @@ public final boolean ready() throws IOException, InterruptedException { if (heap.size() == 0) { return false; } - for(SpanIterator sp: heap) { - sb.append(sp.toString()); - sb.append(","); - total += sp.span.length(); - eq += sp.span.getEq(); + for (SpanIterator sp : heap) { + sb.append(sp.toString()); + sb.append(","); + total += sp.span.length(); + eq += sp.span.getEq(); } LOG.info(outputContext.getInputOutputVertexNames() + ": " + "Heap = " + sb.toString()); return true; - } catch(ExecutionException e) { + } catch (ExecutionException e) { LOG.error("Heap size={}, total={}, eq={}, partition={}, gallop={}, totalItr={}," + " futures.size={}, destVertexName={}", heap.size(), total, eq, partition, gallop, numSpanItr, futures.size(), @@ -1420,21 +1425,21 @@ public final boolean ready() throws IOException, InterruptedException { } private SpanIterator pop() { - if(gallop > 0) { + if (gallop > 0) { gallop--; return horse; } SpanIterator current = heap.pop(); SpanIterator next = heap.peek(); - if(next != null && current != null && - ((Object)horse) == ((Object)current)) { + if (next != null && current != null && + ((Object) horse) == ((Object) current)) { // TODO: a better threshold check than 1 key repeating - gallop = current.bisect(next.getKey(), next.getPartition())-1; + gallop = current.bisect(next.getKey(), next.getPartition()) - 1; } horse = current; return current; } - + public boolean needsRLE() { return (eq > 0.1 * total); } @@ -1450,11 +1455,11 @@ private SpanIterator peek() { public final boolean next() { SpanIterator current = pop(); - if(current != null) { + if (current != null) { partition = current.getPartition(); key.reset(current.getKey()); value.reset(current.getValue()); - if(gallop <= 0) { + if (gallop <= 0) { // since all keys and values are references to the kvbuffer, no more deep copies this.add(current); } else { @@ -1480,9 +1485,11 @@ public Integer peekPartition() { } } - public DataInputBuffer getKey() { return key; } - public DataInputBuffer getValue() { return value; } - public int getPartition() { return partition; } + public DataInputBuffer getKey() {return key;} + + public DataInputBuffer getValue() {return value;} + + public int getPartition() {return partition;} public void close() throws IOException { } @@ -1501,6 +1508,5 @@ public TezRawKeyValueIterator filter(int partition) { partIter.reset(partition); return partIter; } - } } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezIndexRecord.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezIndexRecord.java index 872cd10cf8..5a5a0446e3 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezIndexRecord.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezIndexRecord.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.runtime.library.common.sort.impl; @@ -23,7 +23,7 @@ public class TezIndexRecord { private long rawLength; private long partLength; - public TezIndexRecord() { } + public TezIndexRecord() {} /** * @param startOffset start offset within the data file diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java index e83b34e114..ed5395757a 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -64,80 +64,77 @@ public class TezMerger { private static final Logger LOG = LoggerFactory.getLogger(TezMerger.class); - // Local directories - private static LocalDirAllocator lDirAlloc = - new LocalDirAllocator(TezRuntimeFrameworkConfigs.LOCAL_DIRS); + private static LocalDirAllocator lDirAlloc = + new LocalDirAllocator(TezRuntimeFrameworkConfigs.LOCAL_DIRS); - public static - TezRawKeyValueIterator merge(Configuration conf, FileSystem fs, - SerializationContext serializationContext, - CompressionCodec codec, boolean ifileReadAhead, - int ifileReadAheadLength, int ifileBufferSize, - Path[] inputs, boolean deleteInputs, - int mergeFactor, Path tmpDir, - RawComparator comparator, Progressable reporter, - TezCounter readsCounter, - TezCounter writesCounter, - TezCounter bytesReadCounter, - Progress mergePhase) + public static TezRawKeyValueIterator merge(Configuration conf, FileSystem fs, + SerializationContext serializationContext, + CompressionCodec codec, boolean ifileReadAhead, + int ifileReadAheadLength, int ifileBufferSize, + Path[] inputs, boolean deleteInputs, + int mergeFactor, Path tmpDir, + RawComparator comparator, Progressable reporter, + TezCounter readsCounter, + TezCounter writesCounter, + TezCounter bytesReadCounter, + Progress mergePhase) throws IOException, InterruptedException { - return - new MergeQueue(conf, fs, inputs, deleteInputs, codec, ifileReadAhead, - ifileReadAheadLength, ifileBufferSize, false, comparator, - reporter, null).merge(serializationContext, - mergeFactor, tmpDir, - readsCounter, writesCounter, - bytesReadCounter, - mergePhase); + return + new MergeQueue(conf, fs, inputs, deleteInputs, codec, ifileReadAhead, + ifileReadAheadLength, ifileBufferSize, false, comparator, + reporter, null).merge(serializationContext, + mergeFactor, tmpDir, + readsCounter, writesCounter, + bytesReadCounter, + mergePhase); } // Used by the in-memory merger. - public static - TezRawKeyValueIterator merge(Configuration conf, FileSystem fs, - SerializationContext serializationContext, - List segments, - int mergeFactor, Path tmpDir, - RawComparator comparator, Progressable reporter, - TezCounter readsCounter, - TezCounter writesCounter, - TezCounter bytesReadCounter, - Progress mergePhase) + public static TezRawKeyValueIterator merge(Configuration conf, FileSystem fs, + SerializationContext serializationContext, + List segments, + int mergeFactor, Path tmpDir, + RawComparator comparator, Progressable reporter, + TezCounter readsCounter, + TezCounter writesCounter, + TezCounter bytesReadCounter, + Progress mergePhase) throws IOException, InterruptedException { // Get rid of this ? return merge(conf, fs, serializationContext, segments, mergeFactor, tmpDir, - comparator, reporter, false, readsCounter, writesCounter, bytesReadCounter, - mergePhase); + comparator, reporter, false, readsCounter, writesCounter, bytesReadCounter, + mergePhase); } public static TezRawKeyValueIterator merge(Configuration conf, FileSystem fs, - SerializationContext serializationContext, - List segments, - int mergeFactor, Path tmpDir, - RawComparator comparator, Progressable reporter, - boolean sortSegments, - TezCounter readsCounter, - TezCounter writesCounter, - TezCounter bytesReadCounter, - Progress mergePhase) + SerializationContext serializationContext, + List segments, + int mergeFactor, Path tmpDir, + RawComparator comparator, Progressable reporter, + boolean sortSegments, + TezCounter readsCounter, + TezCounter writesCounter, + TezCounter bytesReadCounter, + Progress mergePhase) throws IOException, InterruptedException { return new MergeQueue(conf, fs, segments, comparator, reporter, - sortSegments, false).merge(serializationContext, mergeFactor, tmpDir, - readsCounter, writesCounter, - bytesReadCounter, mergePhase); + sortSegments, false).merge(serializationContext, mergeFactor, tmpDir, + readsCounter, writesCounter, + bytesReadCounter, mergePhase); } public static TezRawKeyValueIterator merge(Configuration conf, FileSystem fs, - SerializationContext serializationContext, - CompressionCodec codec, - List segments, - int mergeFactor, Path tmpDir, - RawComparator comparator, Progressable reporter, - TezCounter readsCounter, - TezCounter writesCounter, - TezCounter bytesReadCounter, - Progress mergePhase) throws IOException, InterruptedException { + SerializationContext serializationContext, + CompressionCodec codec, + List segments, + int mergeFactor, Path tmpDir, + RawComparator comparator, Progressable reporter, + TezCounter readsCounter, + TezCounter writesCounter, + TezCounter bytesReadCounter, + Progress mergePhase) throws IOException, InterruptedException { return new MergeQueue(conf, fs, segments, comparator, reporter, false, codec, false, false) .merge(serializationContext, mergeFactor, tmpDir, @@ -146,17 +143,17 @@ public static TezRawKeyValueIterator merge(Configuration conf, FileSystem fs, public static TezRawKeyValueIterator merge(Configuration conf, FileSystem fs, - SerializationContext serializationContext, - CompressionCodec codec, - List segments, - int mergeFactor, Path tmpDir, - RawComparator comparator, Progressable reporter, - boolean sortSegments, - boolean considerFinalMergeForProgress, - TezCounter readsCounter, - TezCounter writesCounter, - TezCounter bytesReadCounter, - Progress mergePhase, boolean checkForSameKeys) + SerializationContext serializationContext, + CompressionCodec codec, + List segments, + int mergeFactor, Path tmpDir, + RawComparator comparator, Progressable reporter, + boolean sortSegments, + boolean considerFinalMergeForProgress, + TezCounter readsCounter, + TezCounter writesCounter, + TezCounter bytesReadCounter, + Progress mergePhase, boolean checkForSameKeys) throws IOException, InterruptedException { return new MergeQueue(conf, fs, segments, comparator, reporter, sortSegments, codec, considerFinalMergeForProgress, checkForSameKeys). @@ -169,62 +166,62 @@ TezRawKeyValueIterator merge(Configuration conf, FileSystem fs, public static TezRawKeyValueIterator merge(Configuration conf, FileSystem fs, - SerializationContext serializationContext, - CompressionCodec codec, - List segments, - int mergeFactor, Path tmpDir, - RawComparator comparator, Progressable reporter, - boolean sortSegments, - boolean considerFinalMergeForProgress, - TezCounter readsCounter, - TezCounter writesCounter, - TezCounter bytesReadCounter, - Progress mergePhase) + SerializationContext serializationContext, + CompressionCodec codec, + List segments, + int mergeFactor, Path tmpDir, + RawComparator comparator, Progressable reporter, + boolean sortSegments, + boolean considerFinalMergeForProgress, + TezCounter readsCounter, + TezCounter writesCounter, + TezCounter bytesReadCounter, + Progress mergePhase) throws IOException, InterruptedException { return new MergeQueue(conf, fs, segments, comparator, reporter, - sortSegments, codec, considerFinalMergeForProgress). - merge(serializationContext, mergeFactor, tmpDir, - readsCounter, writesCounter, - bytesReadCounter, - mergePhase); + sortSegments, codec, considerFinalMergeForProgress). + merge(serializationContext, mergeFactor, tmpDir, + readsCounter, writesCounter, + bytesReadCounter, + mergePhase); } public static TezRawKeyValueIterator merge(Configuration conf, FileSystem fs, - SerializationContext serializationContext, - CompressionCodec codec, - List segments, - int mergeFactor, int inMemSegments, Path tmpDir, - RawComparator comparator, Progressable reporter, - boolean sortSegments, - TezCounter readsCounter, - TezCounter writesCounter, - TezCounter bytesReadCounter, - Progress mergePhase) + SerializationContext serializationContext, + CompressionCodec codec, + List segments, + int mergeFactor, int inMemSegments, Path tmpDir, + RawComparator comparator, Progressable reporter, + boolean sortSegments, + TezCounter readsCounter, + TezCounter writesCounter, + TezCounter bytesReadCounter, + Progress mergePhase) throws IOException, InterruptedException { - return new MergeQueue(conf, fs, segments, comparator, reporter, - sortSegments, codec, false).merge(serializationContext, - mergeFactor, inMemSegments, - tmpDir, - readsCounter, writesCounter, - bytesReadCounter, - mergePhase); -} + return new MergeQueue(conf, fs, segments, comparator, reporter, + sortSegments, codec, false).merge(serializationContext, + mergeFactor, inMemSegments, + tmpDir, + readsCounter, writesCounter, + bytesReadCounter, + mergePhase); + } public static void writeFile(TezRawKeyValueIterator records, Writer writer, - Progressable progressable, long recordsBeforeProgress) + Progressable progressable, long recordsBeforeProgress) throws IOException, InterruptedException { long recordCtr = 0; long count = 0; - while(records.next()) { + while (records.next()) { if (records.isSameKey()) { writer.append(IFile.REPEAT_KEY, records.getValue()); count++; } else { writer.append(records.getKey(), records.getValue()); } - + if (((recordCtr++) % recordsBeforeProgress) == 0) { progressable.progress(); if (Thread.currentThread().isInterrupted()) { @@ -296,7 +293,7 @@ boolean inMemory() { return true; } - KeyValueBuffer getKey() { return key; } + KeyValueBuffer getKey() {return key;} DataInputBuffer getValue(DataInputBuffer value) throws IOException { nextRawValue(value); @@ -369,43 +366,43 @@ public static class DiskSegment extends Segment { int bufferSize = -1; public DiskSegment(FileSystem fs, Path file, - CompressionCodec codec, boolean ifileReadAhead, - int ifileReadAheadLength, int bufferSize, boolean preserve) - throws IOException { + CompressionCodec codec, boolean ifileReadAhead, + int ifileReadAheadLength, int bufferSize, boolean preserve) + throws IOException { this(fs, file, codec, ifileReadAhead, ifileReadAheadLength, bufferSize, preserve, null); } public DiskSegment(FileSystem fs, Path file, - CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLenth, - int bufferSize, boolean preserve, TezCounter mergedMapOutputsCounter) - throws IOException { + CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLenth, + int bufferSize, boolean preserve, TezCounter mergedMapOutputsCounter) + throws IOException { this(fs, file, 0, fs.getFileStatus(file).getLen(), codec, ifileReadAhead, ifileReadAheadLenth, bufferSize, preserve, mergedMapOutputsCounter); } public DiskSegment(FileSystem fs, Path file, - long segmentOffset, long segmentLength, - CompressionCodec codec, boolean ifileReadAhead, - int ifileReadAheadLength, int bufferSize, - boolean preserve) throws IOException { + long segmentOffset, long segmentLength, + CompressionCodec codec, boolean ifileReadAhead, + int ifileReadAheadLength, int bufferSize, + boolean preserve) throws IOException { this(fs, file, segmentOffset, segmentLength, codec, ifileReadAhead, ifileReadAheadLength, bufferSize, preserve, null); } public DiskSegment(FileSystem fs, Path file, - long segmentOffset, long segmentLength, CompressionCodec codec, - boolean ifileReadAhead, int ifileReadAheadLength, int bufferSize, - boolean preserve, TezCounter mergedMapOutputsCounter) - throws IOException { + long segmentOffset, long segmentLength, CompressionCodec codec, + boolean ifileReadAhead, int ifileReadAheadLength, int bufferSize, + boolean preserve, TezCounter mergedMapOutputsCounter) + throws IOException { super(null, mergedMapOutputsCounter); this.fs = fs; this.file = file; this.codec = codec; this.preserve = preserve; this.ifileReadAhead = ifileReadAhead; - this.ifileReadAheadLength =ifileReadAheadLength; + this.ifileReadAheadLength = ifileReadAheadLength; this.bufferSize = bufferSize; this.segmentOffset = segmentOffset; @@ -429,7 +426,7 @@ boolean inMemory() { @Override public long getLength() { return (reader == null) ? - segmentLength : reader.getLength(); + segmentLength : reader.getLength(); } @Override @@ -439,6 +436,7 @@ void close() throws IOException { fs.delete(file, false); } } + // This method is used by BackupStore to extract the // absolute position after a reset @Override @@ -461,7 +459,7 @@ void reinitReader(int offset) throws IOException { @VisibleForTesting static class MergeQueue - extends PriorityQueue implements TezRawKeyValueIterator { + extends PriorityQueue implements TezRawKeyValueIterator { final Configuration conf; final FileSystem fs; final CompressionCodec codec; @@ -470,9 +468,9 @@ static class MergeQueue static final int ifileReadAheadLength = TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES_DEFAULT; static final int ifileBufferSize = TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_BUFFER_SIZE_DEFAULT; static final long recordsBeforeProgress = TezRuntimeConfiguration.TEZ_RUNTIME_RECORDS_BEFORE_PROGRESS_DEFAULT; - + List segments = new ArrayList(); - + final RawComparator comparator; private long totalBytesProcessed; @@ -484,35 +482,35 @@ static class MergeQueue private final boolean considerFinalMergeForProgress; final Progressable reporter; - + final DataInputBuffer key = new DataInputBuffer(); final DataInputBuffer value = new DataInputBuffer(); final DataInputBuffer nextKey = new DataInputBuffer(); final DataInputBuffer diskIFileValue = new DataInputBuffer(); - + Segment minSegment; - Comparator segmentComparator = - new Comparator() { - public int compare(Segment o1, Segment o2) { - if (o1.getLength() == o2.getLength()) { - return 0; - } + Comparator segmentComparator = + new Comparator() { + public int compare(Segment o1, Segment o2) { + if (o1.getLength() == o2.getLength()) { + return 0; + } - return o1.getLength() < o2.getLength() ? -1 : 1; - } - }; + return o1.getLength() < o2.getLength() ? -1 : 1; + } + }; KeyState hasNext; DataOutputBuffer prevKey = new DataOutputBuffer(); - public MergeQueue(Configuration conf, FileSystem fs, + public MergeQueue(Configuration conf, FileSystem fs, Path[] inputs, boolean deleteInputs, CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLength, int ifileBufferSize, boolean considerFinalMergeForProgress, - RawComparator comparator, Progressable reporter, - TezCounter mergedMapOutputsCounter) - throws IOException { + RawComparator comparator, Progressable reporter, + TezCounter mergedMapOutputsCounter) + throws IOException { this.conf = conf; this.checkForSameKeys = true; // this.recordsBeforeProgress = @@ -523,42 +521,42 @@ public MergeQueue(Configuration conf, FileSystem fs, this.comparator = comparator; this.reporter = reporter; this.considerFinalMergeForProgress = considerFinalMergeForProgress; - + for (Path file : inputs) { if (LOG.isTraceEnabled()) { LOG.trace("MergeQ: adding: " + file); } segments.add(new DiskSegment(fs, file, codec, ifileReadAhead, - ifileReadAheadLength, ifileBufferSize, - !deleteInputs, - (file.toString().endsWith( - Constants.MERGED_OUTPUT_PREFIX) ? - null : mergedMapOutputsCounter))); + ifileReadAheadLength, ifileBufferSize, + !deleteInputs, + (file.toString().endsWith( + Constants.MERGED_OUTPUT_PREFIX) ? + null : mergedMapOutputsCounter))); } - + // Sort segments on file-lengths - Collections.sort(segments, segmentComparator); + Collections.sort(segments, segmentComparator); } - + public MergeQueue(Configuration conf, FileSystem fs, - List segments, RawComparator comparator, - Progressable reporter, boolean sortSegments, boolean considerFinalMergeForProgress) { + List segments, RawComparator comparator, + Progressable reporter, boolean sortSegments, boolean considerFinalMergeForProgress) { this(conf, fs, segments, comparator, reporter, sortSegments, null, considerFinalMergeForProgress); } public MergeQueue(Configuration conf, FileSystem fs, - List segments, RawComparator comparator, - Progressable reporter, boolean sortSegments, CompressionCodec codec, - boolean considerFinalMergeForProgress) { + List segments, RawComparator comparator, + Progressable reporter, boolean sortSegments, CompressionCodec codec, + boolean considerFinalMergeForProgress) { this(conf, fs, segments, comparator, reporter, sortSegments, codec, considerFinalMergeForProgress, true); } public MergeQueue(Configuration conf, FileSystem fs, - List segments, RawComparator comparator, - Progressable reporter, boolean sortSegments, CompressionCodec codec, - boolean considerFinalMergeForProgress, boolean checkForSameKeys) { + List segments, RawComparator comparator, + Progressable reporter, boolean sortSegments, CompressionCodec codec, + boolean considerFinalMergeForProgress, boolean checkForSameKeys) { this.conf = conf; this.fs = fs; this.comparator = comparator; @@ -574,7 +572,7 @@ public MergeQueue(Configuration conf, FileSystem fs, public void close() throws IOException { Segment segment; - while((segment = pop()) != null) { + while ((segment = pop()) != null) { segment.close(); } } @@ -592,7 +590,7 @@ private void populatePreviousKey() throws IOException { BufferUtils.copy(key, prevKey); } - private void adjustPriorityQueue(Segment reader) throws IOException{ + private void adjustPriorityQueue(Segment reader) throws IOException { long startPos = reader.getPosition(); if (checkForSameKeys) { if (hasNext == null) { @@ -623,11 +621,11 @@ private void adjustPriorityQueue(Segment reader) throws IOException{ if (hasNext == KeyState.NEW_KEY) { adjustTop(); compareKeyWithNextTopKey(reader); - } else if(hasNext == KeyState.NO_KEY) { + } else if (hasNext == KeyState.NO_KEY) { pop(); reader.close(); compareKeyWithNextTopKey(null); - } else if(hasNext == KeyState.SAME_KEY) { + } else if (hasNext == KeyState.SAME_KEY) { // do not rebalance the priority queue } } @@ -694,16 +692,17 @@ int compare(KeyValueBuffer nextKey, DataOutputBuffer buf2) { } protected boolean lessThan(Object a, Object b) { - KeyValueBuffer key1 = ((Segment)a).getKey(); - KeyValueBuffer key2 = ((Segment)b).getKey(); + KeyValueBuffer key1 = ((Segment) a).getKey(); + KeyValueBuffer key2 = ((Segment) b).getKey(); int s1 = key1.getPosition(); int l1 = key1.getLength(); int s2 = key2.getPosition(); - int l2 = key2.getLength();; + int l2 = key2.getLength(); + ; return comparator.compare(key1.getData(), s1, l1, key2.getData(), s2, l2) < 0; } - + public TezRawKeyValueIterator merge(SerializationContext serializationContext, int factor, Path tmpDir, TezCounter readsCounter, @@ -712,15 +711,15 @@ public TezRawKeyValueIterator merge(SerializationContext serializationContext, Progress mergePhase) throws IOException, InterruptedException { return merge(serializationContext, factor, 0, tmpDir, - readsCounter, writesCounter, bytesReadCounter, mergePhase); + readsCounter, writesCounter, bytesReadCounter, mergePhase); } TezRawKeyValueIterator merge(SerializationContext serializationContext, - int factor, int inMem, Path tmpDir, - TezCounter readsCounter, - TezCounter writesCounter, - TezCounter bytesReadCounter, - Progress mergePhase) + int factor, int inMem, Path tmpDir, + TezCounter readsCounter, + TezCounter writesCounter, + TezCounter bytesReadCounter, + Progress mergePhase) throws IOException, InterruptedException { if (segments.size() == 0) { LOG.info("Nothing to merge. Returning an empty iterator"); @@ -745,9 +744,9 @@ TezRawKeyValueIterator merge(SerializationContext serializationContext, long totalBytes = computeBytesInMerges(segments, factor, inMem, considerFinalMergeForProgress); if (totalBytes != 0) { - progPerByte = 1.0f / (float)totalBytes; + progPerByte = 1.0f / (float) totalBytes; } - + //create the MergeStreams from the sorted map created in the constructor //and dump the final output to a file do { @@ -759,15 +758,15 @@ TezRawKeyValueIterator merge(SerializationContext serializationContext, factor += inMem; } List segmentsToMerge = - new ArrayList(); + new ArrayList(); int segmentsConsidered = 0; int numSegmentsToConsider = factor; long startBytes = 0; // starting bytes of segments of this merge while (true) { //extract the smallest 'factor' number of segments //Call cleanup on the empty segments (no key/value data) - List mStream = - getSegmentDescriptors(numSegmentsToConsider); + List mStream = + getSegmentDescriptors(numSegmentsToConsider); for (Segment segment : mStream) { // Initialize the segment at the last possible moment; // this helps in ensuring we don't use buffers until we need them @@ -776,20 +775,19 @@ TezRawKeyValueIterator merge(SerializationContext serializationContext, long startPos = segment.getPosition(); boolean hasNext = segment.nextRawKey(nextKey); long endPos = segment.getPosition(); - + if (hasNext) { startBytes += endPos - startPos; segmentsToMerge.add(segment); segmentsConsidered++; - } - else { // Empty segments. Can this be avoided altogether ? + } else { // Empty segments. Can this be avoided altogether ? segment.close(); numSegments--; //we ignore this segment for the merge } } //if we have the desired number of segments //or looked at all available segments, we break - if (segmentsConsidered == factor || + if (segmentsConsidered == factor || segments.size() == 0) { break; } @@ -797,14 +795,14 @@ TezRawKeyValueIterator merge(SerializationContext serializationContext, // Get the correct # of segments in case some of them were empty. numSegmentsToConsider = factor - segmentsConsidered; } - + //feed the streams to the priority queue initialize(segmentsToMerge.size()); clear(); for (Segment segment : segmentsToMerge) { put(segment); } - + //if we have lesser number of segments remaining, then just return the //iterator, else do another single level merge if (numSegments <= factor) { // Will always kick in if only in-mem segments are provided. @@ -821,8 +819,8 @@ TezRawKeyValueIterator merge(SerializationContext serializationContext, } } if (totalBytes != 0) //being paranoid - progPerByte = 1.0f / (float)totalBytes; - + progPerByte = 1.0f / (float) totalBytes; + totalBytesProcessed += startBytes; if (totalBytes != 0) mergeProgress.set(totalBytesProcessed * progPerByte); @@ -845,24 +843,24 @@ TezRawKeyValueIterator merge(SerializationContext serializationContext, " intermediate segments out of a total of " + (segments.size() + segmentsToMerge.size())); } - + long bytesProcessedInPrevMerges = totalBytesProcessed; totalBytesProcessed += startBytes; //we want to spread the creation of temp files on multiple disks if //available under the space constraints - long approxOutputSize = 0; + long approxOutputSize = 0; for (Segment s : segmentsToMerge) { - approxOutputSize += s.getLength() + - ChecksumFileSystem.getApproxChkSumLength( - s.getLength()); + approxOutputSize += s.getLength() + + ChecksumFileSystem.getApproxChkSumLength( + s.getLength()); } - Path tmpFilename = - new Path(tmpDir, "intermediate").suffix("." + passNo); + Path tmpFilename = + new Path(tmpDir, "intermediate").suffix("." + passNo); - Path outputFile = lDirAlloc.getLocalPathForWrite( - tmpFilename.toString(), - approxOutputSize, conf); + Path outputFile = lDirAlloc.getLocalPathForWrite( + tmpFilename.toString(), + approxOutputSize, conf); // TODO Would it ever make sense to make this an in-memory writer ? // Merging because of too many disk segments - might fit in memory. @@ -873,45 +871,45 @@ TezRawKeyValueIterator merge(SerializationContext serializationContext, writeFile(this, writer, reporter, recordsBeforeProgress); writer.close(); - + //we finished one single level merge; now clean up the priority //queue this.close(); // Add the newly create segment to the list of segments to be merged - Segment tempSegment = - new DiskSegment(fs, outputFile, codec, ifileReadAhead, - ifileReadAheadLength, ifileBufferSize, false); + Segment tempSegment = + new DiskSegment(fs, outputFile, codec, ifileReadAhead, + ifileReadAheadLength, ifileBufferSize, false); // Insert new merged segment into the sorted list int pos = Collections.binarySearch(segments, tempSegment, - segmentComparator); + segmentComparator); if (pos < 0) { // binary search failed. So position to be inserted at is -pos-1 - pos = -pos-1; + pos = -pos - 1; } segments.add(pos, tempSegment); numSegments = segments.size(); - + // Subtract the difference between expected size of new segment and // actual size of new segment(Expected size of new segment is // inputBytesOfThisMerge) from totalBytes. Expected size and actual // size will match(almost) if combiner is not called in merge. long inputBytesOfThisMerge = totalBytesProcessed - - bytesProcessedInPrevMerges; + bytesProcessedInPrevMerges; totalBytes -= inputBytesOfThisMerge - tempSegment.getLength(); if (totalBytes != 0) { - progPerByte = 1.0f / (float)totalBytes; + progPerByte = 1.0f / (float) totalBytes; } - + passNo++; } //we are worried about only the first pass merge factor. So reset the //factor to what it originally was factor = origFactor; - } while(true); + } while (true); } - + /** * Determine the number of segments to merge in a given pass. Assuming more * than factor segments, the first pass should attempt to bring the total @@ -920,14 +918,14 @@ TezRawKeyValueIterator merge(SerializationContext serializationContext, */ private static int getPassFactor(int factor, int passNo, int numSegments) { // passNo > 1 in the OR list - is that correct ? - if (passNo > 1 || numSegments <= factor || factor == 1) + if (passNo > 1 || numSegments <= factor || factor == 1) return factor; int mod = (numSegments - 1) % (factor - 1); if (mod == 0) return factor; return mod + 1; } - + /** Return (& remove) the requested number of segment descriptors from the * sorted map. */ @@ -944,7 +942,7 @@ private List getSegmentDescriptors(int numDescriptors) { subList.clear(); return subListCopy; } - + /** * Compute expected size of input bytes to merges, will be used in * calculating mergeProgress. This simulates the above merge() method and @@ -963,13 +961,13 @@ static long computeBytesInMerges(List segments, int factor, int inMem, // factor for 1st pass int f = getPassFactor(factor, 1, n) + inMem; n = numSegments; - + for (int i = 0; i < numSegments; i++) { // Not handling empty segments here assuming that it would not affect // much in calculation of mergeProgress. segmentSizes[i] = segments.get(i).getLength(); } - + // If includeFinalMerge is true, allow the following while loop iterate // for 1 more iteration. This is to include final merge as part of the // computation of expected input bytes of merges @@ -986,11 +984,11 @@ static long computeBytesInMerges(List segments, int factor, int inMem, mergedSize += segmentSizes[offset + j]; } totalBytes += mergedSize; - + // insert new size into the sorted list int pos = Arrays.binarySearch(segmentSizes, offset, offset + n, mergedSize); if (pos < 0) { - pos = -pos-1; + pos = -pos - 1; } if (pos < offset + f) { // Insert at the beginning @@ -1027,7 +1025,7 @@ static long computeBytesInMerges(List segments, int factor, int inMem, offset = 0; } } - n -= f - 1; + n -= f - 1; f = factor; } @@ -1060,7 +1058,6 @@ public boolean hasNext() throws IOException { return true; } - } private static class EmptyIterator implements TezRawKeyValueIterator { diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezRawKeyValueIterator.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezRawKeyValueIterator.java index 683c9b9ce9..29af5b1d81 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezRawKeyValueIterator.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezRawKeyValueIterator.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,25 +31,25 @@ @InterfaceAudience.Private @InterfaceStability.Unstable public interface TezRawKeyValueIterator { - /** + /** * Gets the current raw key. - * + * * @return Gets the current raw key as a DataInputBuffer * @throws IOException */ DataInputBuffer getKey() throws IOException; - - /** + + /** * Gets the current raw value. - * + * * @return Gets the current raw value as a DataInputBuffer * @throws IOException */ DataInputBuffer getValue() throws IOException; - - /** + + /** * Sets up the current key and value (for getKey and getValue). - * + * * @return true if there exists a key/value, * false otherwise. * @throws IOException @@ -65,13 +65,13 @@ public interface TezRawKeyValueIterator { */ boolean hasNext() throws IOException; - /** + /** * Closes the iterator so that the underlying streams can be closed. - * + * * @throws IOException */ void close() throws IOException; - + /** Gets the Progress object; this has a float (0.0 - 1.0) * indicating the bytes processed by the iterator so far */ diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezSpillRecord.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezSpillRecord.java index 1c9edeead7..4c73c78ac4 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezSpillRecord.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezSpillRecord.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -58,18 +58,18 @@ public TezSpillRecord(Path indexFileName, FileSystem fs) throws IOException { } public TezSpillRecord(Path indexFileName, FileSystem fs, String expectedIndexOwner) - throws IOException { + throws IOException { this(indexFileName, fs, new PureJavaCrc32(), expectedIndexOwner); } public TezSpillRecord(Path indexFileName, FileSystem rfs, Checksum crc, - String expectedIndexOwner) + String expectedIndexOwner) throws IOException { final FSDataInputStream in = rfs.open(indexFileName); try { final long length = rfs.getFileStatus(indexFileName).getLen(); - final int partitions = + final int partitions = (int) length / Constants.MAP_OUTPUT_INDEX_RECORD_LENGTH; final int size = partitions * Constants.MAP_OUTPUT_INDEX_RECORD_LENGTH; @@ -80,7 +80,7 @@ public TezSpillRecord(Path indexFileName, FileSystem rfs, Checksum crc, IOUtils.readFully(chk, buf.array(), 0, size); if (chk.getChecksum().getValue() != in.readLong()) { throw new ChecksumException("Checksum error reading spill index: " + - indexFileName, -1); + indexFileName, -1); } } else { IOUtils.readFully(in, buf.array(), 0, size); @@ -104,7 +104,7 @@ public int size() { public TezIndexRecord getIndex(int partition) { final int pos = partition * Constants.MAP_OUTPUT_INDEX_RECORD_LENGTH / 8; return new TezIndexRecord(entries.get(pos), entries.get(pos + 1), - entries.get(pos + 2)); + entries.get(pos + 2)); } /** diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java index 7c678749b2..4fbf271311 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.runtime.library.common.sort.impl.dflt; @@ -67,11 +67,11 @@ @SuppressWarnings({"unchecked", "rawtypes"}) public final class DefaultSorter extends ExternalSorter implements IndexedSortable { - + private static final Logger LOG = LoggerFactory.getLogger(DefaultSorter.class); // TODO NEWTEZ Progress reporting to Tez framework. (making progress vs %complete) - + /** * The size of each record in the index file for the map-outputs. */ @@ -91,7 +91,7 @@ public final class DefaultSorter extends ExternalSorter implements IndexedSortab int bufmark; // marks end of record int bufindex; // marks end of collected int bufvoid; // marks the point where we should stop - // reading at the end of the buffer + // reading at the end of the buffer private byte[] kvbuffer; // main output buffer private final byte[] b0 = new byte[0]; @@ -121,7 +121,7 @@ public final class DefaultSorter extends ExternalSorter implements IndexedSortab private final String auxiliaryService; final ArrayList indexCacheList = - new ArrayList(); + new ArrayList(); private final int indexCacheMemoryLimit; private int totalIndexCacheMemory; @@ -130,9 +130,8 @@ public final class DefaultSorter extends ExternalSorter implements IndexedSortab public static final int MAX_IO_SORT_MB = 1800; - public DefaultSorter(OutputContext outputContext, Configuration conf, int numOutputs, - long initialMemoryAvailable) throws IOException { + long initialMemoryAvailable) throws IOException { super(outputContext, conf, numOutputs, initialMemoryAvailable); deflater = TezCommonUtils.newBestCompressionDeflater(); // sanity checks @@ -146,7 +145,7 @@ public DefaultSorter(OutputContext outputContext, Configuration conf, int numOut + " should be greater than 0 and less than or equal to 1"); indexCacheMemoryLimit = this.conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES, - TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES_DEFAULT); + TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES_DEFAULT); boolean confPipelinedShuffle = this.conf.getBoolean(TezRuntimeConfiguration .TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED, TezRuntimeConfiguration @@ -166,14 +165,14 @@ public DefaultSorter(OutputContext outputContext, Configuration conf, int numOut kvbuffer = new byte[maxMemUsage]; bufvoid = kvbuffer.length; kvmeta = ByteBuffer.wrap(kvbuffer) - .order(ByteOrder.nativeOrder()) - .asIntBuffer(); + .order(ByteOrder.nativeOrder()) + .asIntBuffer(); setEquator(0); bufstart = bufend = bufindex = equator; kvstart = kvend = kvindex; maxRec = kvmeta.capacity() / NMETA; - softLimit = (int)(kvbuffer.length * spillper); + softLimit = (int) (kvbuffer.length * spillper); bufferRemaining = softLimit; if (LOG.isInfoEnabled()) { LOG.info(outputContext.getDestinationVertexName() + ": " @@ -247,17 +246,17 @@ public void write(Object key, Object value) * storage to store one METADATA. */ synchronized void collect(Object key, Object value, final int partition - ) throws IOException { + ) throws IOException { if (key.getClass() != serializationContext.getKeyClass()) { throw new IOException("Type mismatch in key from map: expected " - + serializationContext.getKeyClass().getName() + ", received " - + key.getClass().getName()); + + serializationContext.getKeyClass().getName() + ", received " + + key.getClass().getName()); } if (value.getClass() != serializationContext.getValueClass()) { throw new IOException("Type mismatch in value from map: expected " - + serializationContext.getValueClass().getName() + ", received " - + value.getClass().getName()); + + serializationContext.getValueClass().getName() + ", received " + + value.getClass().getName()); } if (partition < 0 || partition >= partitions) { throw new IOException("Illegal partition for " + key + " (" + @@ -292,8 +291,8 @@ synchronized void collect(Object key, Object value, final int partition // be possible for metadata alignment to hit spill pcnt startSpill(); final int avgRec = (int) - (mapOutputByteCounter.getValue() / - mapOutputRecordCounter.getValue()); + (mapOutputByteCounter.getValue() / + mapOutputRecordCounter.getValue()); // leave at least half the split buffer for serialization data // ensure that kvindex >= bufindex final int distkvi = distanceTo(bufindex, kvbidx); @@ -307,10 +306,10 @@ synchronized void collect(Object key, Object value, final int partition * overflow) */ final int newPos = (bufindex + - Math.max(2 * METASIZE - 1, - Math.min(distkvi / 2, - distkvi / (METASIZE + avgRec) * METASIZE))) - % kvbuffer.length; + Math.max(2 * METASIZE - 1, + Math.min(distkvi / 2, + distkvi / (METASIZE + avgRec) * METASIZE))) + % kvbuffer.length; setEquator(newPos); bufmark = bufindex = newPos; final int serBound = 4 * kvend; @@ -321,10 +320,10 @@ synchronized void collect(Object key, Object value, final int partition // metadata max distanceTo(bufend, newPos), Math.min( - // serialization max - distanceTo(newPos, serBound), - // soft limit - softLimit)) - 2 * METASIZE; + // serialization max + distanceTo(newPos, serBound), + // soft limit + softLimit)) - 2 * METASIZE; } } } while (false); @@ -368,7 +367,7 @@ synchronized void collect(Object key, Object value, final int partition kvmeta.put(kvindex + VALSTART, valstart); kvmeta.put(kvindex + VALLEN, distanceTo(valstart, valend)); // advance kvindex - kvindex = (int)(((long)kvindex - NMETA + kvmeta.capacity()) % kvmeta.capacity()); + kvindex = (int) (((long) kvindex - NMETA + kvmeta.capacity()) % kvmeta.capacity()); totalKeys++; } catch (MapBufferTooSmallException e) { LOG.info( @@ -410,14 +409,14 @@ private void resetSpill() { kvstart = kvend = (int) (((long) aligned - METASIZE + kvbuffer.length) % kvbuffer.length) / 4; if (LOG.isInfoEnabled()) { LOG.info(outputContext.getInputOutputVertexNames() + ": " + "(RESET) equator " + e + " kv " + kvstart + "(" + - (kvstart * 4) + ")" + " kvi " + kvindex + "(" + (kvindex * 4) + ")"); + (kvstart * 4) + ")" + " kvi " + kvindex + "(" + (kvindex * 4) + ")"); } } /** * Compute the distance in bytes between two indices in the serialization * buffer. - * @see #distanceTo(int,int,int) + * @see #distanceTo(int, int, int) */ final int distanceTo(final int i, final int j) { return distanceTo(i, j, kvbuffer.length); @@ -429,8 +428,8 @@ final int distanceTo(final int i, final int j) { */ int distanceTo(final int i, final int j, final int mod) { return i <= j - ? j - i - : mod - i + j; + ? j - i + : mod - i + j; } /** @@ -439,7 +438,6 @@ int distanceTo(final int i, final int j, final int mod) { */ int offsetFor(int metapos) { return (metapos % maxRec) * NMETA; - } /** @@ -470,6 +468,7 @@ public int compare(final int mi, final int mj) { } final byte META_BUFFER_TMP[] = new byte[METASIZE]; + /** * Swap metadata for items i,j * @see IndexedSortable#swap @@ -519,7 +518,7 @@ protected void shiftBufferedKey() throws IOException { final int kvbidx = 4 * kvindex; final int kvbend = 4 * kvend; final int avail = - Math.min(distanceTo(0, kvbidx), distanceTo(0, kvbend)); + Math.min(distanceTo(0, kvbidx), distanceTo(0, kvbend)); if (bufindex + headbytelen < avail) { System.arraycopy(kvbuffer, 0, kvbuffer, headbytelen, bufindex); System.arraycopy(kvbuffer, bufvoid, kvbuffer, 0, headbytelen); @@ -541,7 +540,7 @@ public class Buffer extends OutputStream { @Override public void write(int v) throws IOException { - scratch[0] = (byte)v; + scratch[0] = (byte) v; write(scratch, 0, 1); } @@ -583,8 +582,8 @@ public void write(byte b[], int off, int len) // either the metadata or the current write. Note that collect // ensures its metadata requirement with a zero-length write blockwrite = distkvi <= distkve - ? distkvi <= len + 2 * METASIZE - : distkve <= len || distanceTo(bufend, kvbidx) < 2 * METASIZE; + ? distkvi <= len + 2 * METASIZE + : distkve <= len || distanceTo(bufend, kvbidx) < 2 * METASIZE; if (!spillInProgress) { if (blockwrite) { @@ -629,8 +628,8 @@ public void write(byte b[], int off, int len) } } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new IOInterruptedException( - "Buffer interrupted while waiting for the writer", e); + throw new IOInterruptedException( + "Buffer interrupted while waiting for the writer", e); } } } while (blockwrite); @@ -687,7 +686,7 @@ public void flush() throws IOException { } try { interruptSpillThread(); - } catch(IOException e) { + } catch (IOException e) { //safe to ignore } return; @@ -833,8 +832,8 @@ private void startSpill() { if (LOG.isInfoEnabled()) { LOG.info(outputContext.getInputOutputVertexNames() + ": Spilling map output." + "bufstart=" + bufstart + ", bufend = " + bufmark + ", bufvoid = " + bufvoid - +"; kvstart=" + kvstart + "(" + (kvstart * 4) + ")" - +", kvend = " + kvend + "(" + (kvend * 4) + ")" + + "; kvstart=" + kvstart + "(" + (kvstart * 4) + ")" + + ", kvend = " + kvend + "(" + (kvend * 4) + ")" + ", length = " + (distanceTo(kvend, kvstart, kvmeta.capacity()) + 1) + "/" + maxRec); } spillThread.setTotalKeysProcessed(sameKey, totalKeys); @@ -848,8 +847,8 @@ int getMetaStart() { int getMetaEnd() { return 1 + // kvend is a valid record (kvstart >= kvend - ? kvstart - : kvmeta.capacity() + kvstart) / NMETA; + ? kvstart + : kvmeta.capacity() + kvstart) / NMETA; } private boolean isRLENeeded(long sameKeyCount, long totalKeysCount) { @@ -887,7 +886,7 @@ protected void spill(int mstart, int mend, long sameKeyCount, long totalKeysCoun final long size = (bufend >= bufstart ? bufend - bufstart : (bufvoid - bufend) + bufstart) + - partitions * APPROX_HEADER_LENGTH; + partitions * APPROX_HEADER_LENGTH; FSDataOutputStream out = null; try { // create spill file @@ -928,14 +927,14 @@ protected void spill(int mstart, int mend, long sameKeyCount, long totalKeysCoun int spstart = spindex; while (spindex < mend && kvmeta.get(offsetFor(spindex) - + PARTITION) == i) { + + PARTITION) == i) { ++spindex; } // Note: we would like to avoid the combiner if we've fewer // than some threshold of records for a partition if (spstart != spindex) { TezRawKeyValueIterator kvIter = - new MRResultIterator(spstart, spindex); + new MRResultIterator(spstart, spindex); if (LOG.isDebugEnabled()) { LOG.debug(outputContext.getInputOutputVertexNames() + ": " + "Running combine processor"); } @@ -974,10 +973,10 @@ protected void spill(int mstart, int mend, long sameKeyCount, long totalKeysCoun } else { indexCacheList.add(spillRec); totalIndexCacheMemory += - spillRec.size() * MAP_OUTPUT_INDEX_RECORD_LENGTH; + spillRec.size() * MAP_OUTPUT_INDEX_RECORD_LENGTH; } LOG.info(outputContext.getInputOutputVertexNames() + ": " + "Finished spill " + numSpills - + " at " + filename.toString()); + + " at " + filename.toString()); ++numSpills; if (!isFinalMergeEnabled()) { numShuffleChunks.setValue(numSpills); @@ -1026,8 +1025,8 @@ private void spillSingleRecord(final Object key, final Object value, // compression mapOutputByteCounter.increment(out.getPos() - recordStart); } - long rawLength =0; - long partLength =0; + long rawLength = 0; + long partLength = 0; if (writer != null) { writer.close(); rawLength = writer.getRawLength(); @@ -1055,7 +1054,7 @@ private void spillSingleRecord(final Object key, final Object value, } else { indexCacheList.add(spillRec); totalIndexCacheMemory += - spillRec.size() * MAP_OUTPUT_INDEX_RECORD_LENGTH; + spillRec.size() * MAP_OUTPUT_INDEX_RECORD_LENGTH; } ++numSpills; if (!isFinalMergeEnabled()) { @@ -1109,7 +1108,7 @@ public void reset(byte[] buffer, int start, int length) { this.buffer = new byte[this.length]; final int taillen = bufvoid - start; System.arraycopy(buffer, start, this.buffer, 0, taillen); - System.arraycopy(buffer, 0, this.buffer, taillen, length-taillen); + System.arraycopy(buffer, 0, this.buffer, taillen, length - taillen); this.start = 0; } @@ -1126,6 +1125,7 @@ protected class MRResultIterator implements TezRawKeyValueIterator { private final InMemValBytes vbytes = createInMemValBytes(); private final int end; private int current; + public MRResultIterator(int start, int end) { this.end = end; current = start - 1; @@ -1133,22 +1133,25 @@ public MRResultIterator(int start, int end) { @Override public boolean hasNext() throws IOException { - return (current + 1) < end; + return (current + 1) < end; } public boolean next() throws IOException { return ++current < end; } + public DataInputBuffer getKey() throws IOException { final int kvoff = offsetFor(current); keybuf.reset(kvbuffer, kvmeta.get(kvoff + KEYSTART), kvmeta.get(kvoff + VALSTART) - kvmeta.get(kvoff + KEYSTART)); return keybuf; } + public DataInputBuffer getValue() throws IOException { getVBytesForOffset(offsetFor(current), vbytes); return vbytes; } + public Progress getProgress() { return null; } @@ -1158,11 +1161,11 @@ public boolean isSameKey() throws IOException { return false; } - public void close() { } + public void close() {} } private void maybeSendEventForSpill(List events, boolean isLastEvent, - TezSpillRecord spillRecord, int index, boolean sendEvent) throws IOException { + TezSpillRecord spillRecord, int index, boolean sendEvent) throws IOException { if (isFinalMergeEnabled()) { return; } @@ -1186,7 +1189,7 @@ private void maybeAddEventsForSpills() throws IOException { return; } List events = Lists.newLinkedList(); - for(int i=0; i events = Lists.newLinkedList(); //Since there is only one spill, spill record would be present in cache. spillRecord = indexCacheList.get(0); - Path indexPath = mapOutputFile.getSpillIndexFileForWrite(numSpills-1, partitions * + Path indexPath = mapOutputFile.getSpillIndexFileForWrite(numSpills - 1, partitions * MAP_OUTPUT_INDEX_RECORD_LENGTH); spillRecord.writeToFile(indexPath, conf, localFs); maybeSendEventForSpill(events, true, spillRecord, 0, true); @@ -1243,7 +1246,7 @@ private void mergeParts() throws IOException, InterruptedException { //No need to populate finalIndexFile, finalOutputFile etc when finalMerge is disabled } if (spillRecord != null && reportPartitionStats()) { - for(int i=0; i < spillRecord.size(); i++) { + for (int i = 0; i < spillRecord.size(); i++) { partitionStats[i] += spillRecord.getIndex(i).getPartLength(); } } @@ -1320,8 +1323,7 @@ private void mergeParts() throws IOException, InterruptedException { } numShuffleChunks.setValue(numSpills); return; - } - else { + } else { final TezSpillRecord spillRec = new TezSpillRecord(partitions); for (int parts = 0; parts < partitions; parts++) { boolean shouldWrite = false; @@ -1334,9 +1336,9 @@ private void mergeParts() throws IOException, InterruptedException { if (indexRecord.hasData() || !sendEmptyPartitionDetails) { shouldWrite = true; DiskSegment s = - new DiskSegment(rfs, filename[i], indexRecord.getStartOffset(), - indexRecord.getPartLength(), codec, ifileReadAhead, - ifileReadAheadLength, ifileBufferSize, true); + new DiskSegment(rfs, filename[i], indexRecord.getStartOffset(), + indexRecord.getPartLength(), codec, ifileReadAhead, + ifileReadAheadLength, ifileBufferSize, true); segmentList.add(s); } if (LOG.isDebugEnabled()) { @@ -1355,13 +1357,13 @@ private void mergeParts() throws IOException, InterruptedException { boolean sortSegments = segmentList.size() > mergeFactor; //merge TezRawKeyValueIterator kvIter = TezMerger.merge(conf, rfs, - serializationContext, codec, - segmentList, mergeFactor, - new Path(taskIdentifier), - (RawComparator)ConfigUtils.getIntermediateOutputKeyComparator(conf), - progressable, sortSegments, true, - null, spilledRecordsCounter, additionalSpillBytesRead, - null); // Not using any Progress in TezMerger. Should just work. + serializationContext, codec, + segmentList, mergeFactor, + new Path(taskIdentifier), + (RawComparator) ConfigUtils.getIntermediateOutputKeyComparator(conf), + progressable, sortSegments, true, + null, spilledRecordsCounter, additionalSpillBytesRead, + null); // Not using any Progress in TezMerger. Should just work. //write merged output to disk long segmentStart = finalOut.getPos(); @@ -1372,20 +1374,20 @@ private void mergeParts() throws IOException, InterruptedException { serializationContext.getValSerialization(), finalOut, serializationContext.getKeyClass(), serializationContext.getValueClass(), codec, spilledRecordsCounter, null); - if (combiner == null || numSpills < minSpillsForCombine) { - TezMerger.writeFile(kvIter, writer, - progressable, TezRuntimeConfiguration.TEZ_RUNTIME_RECORDS_BEFORE_PROGRESS_DEFAULT); - } else { - runCombineProcessor(kvIter, writer); + if (combiner == null || numSpills < minSpillsForCombine) { + TezMerger.writeFile(kvIter, writer, + progressable, TezRuntimeConfiguration.TEZ_RUNTIME_RECORDS_BEFORE_PROGRESS_DEFAULT); + } else { + runCombineProcessor(kvIter, writer); + } + writer.close(); + rawLength = writer.getRawLength(); + partLength = writer.getCompressedLength(); } - writer.close(); - rawLength = writer.getRawLength(); - partLength = writer.getCompressedLength(); - } - outputBytesWithOverheadCounter.increment(rawLength); - // record offsets - final TezIndexRecord rec = - new TezIndexRecord(segmentStart, rawLength, partLength); + outputBytesWithOverheadCounter.increment(rawLength); + // record offsets + final TezIndexRecord rec = + new TezIndexRecord(segmentStart, rawLength, partLength); spillRec.putIndex(rec, parts); if (reportPartitionStats()) { partitionStats[parts] += partLength; @@ -1394,8 +1396,8 @@ private void mergeParts() throws IOException, InterruptedException { numShuffleChunks.setValue(1); //final merge has happened spillRec.writeToFile(finalIndexFile, conf, localFs); finalOut.close(); - for(int i = 0; i < numSpills; i++) { - rfs.delete(filename[i],true); + for (int i = 0; i < numSpills; i++) { + rfs.delete(filename[i], true); } } } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/package-info.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/package-info.java index 3cf4e9cc3c..6fe7adfdfe 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/package-info.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/package-info.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,4 +19,4 @@ @Private package org.apache.tez.runtime.library.common.sort.impl.dflt; -import org.apache.hadoop.classification.InterfaceAudience.Private; \ No newline at end of file +import org.apache.hadoop.classification.InterfaceAudience.Private; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/package-info.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/package-info.java index e9737bc658..1a2cabf35a 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/package-info.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/package-info.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,4 +19,4 @@ @Private package org.apache.tez.runtime.library.common.sort.impl; -import org.apache.hadoop.classification.InterfaceAudience.Private; \ No newline at end of file +import org.apache.hadoop.classification.InterfaceAudience.Private; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutput.java index 414f3d060f..f221be72a7 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -67,7 +67,7 @@ public TezTaskOutput(Configuration conf, String uniqueId, int dagID) { /** * Create a local output file name. This method is meant to be used *only* if * the size of the file is not know up front. - * + * * @return path the path to write to * @throws IOException */ @@ -110,7 +110,6 @@ public TezTaskOutput(Configuration conf, String uniqueId, int dagID) { public abstract Path getSpillFileForWrite(int spillNumber, long size) throws IOException; - /** * Create a local output spill index file name. * @@ -131,7 +130,7 @@ public abstract Path getSpillIndexFileForWrite(int spillNumber, long size) * @throws IOException */ public abstract Path getInputFileForWrite(int srcIdentifier, - int spillNum, long size) throws IOException; + int spillNum, long size) throws IOException; /** * Construct a spill file name, given a spill number @@ -141,5 +140,4 @@ public abstract Path getInputFileForWrite(int srcIdentifier, * @return a spill file name independent of the unique identifier and local directories */ public abstract String getSpillFileName(int srcId, int spillNum); - } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutputFiles.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutputFiles.java index 3fb90865d1..f9d76af8b4 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutputFiles.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/TezTaskOutputFiles.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -58,7 +58,7 @@ public TezTaskOutputFiles(Configuration conf, String uniqueId, int dagID) { ${yarnLocalDir}/usercache/${user}/appcache/${applicationId}. (Referred to as ${appDir} later in the docs */ private final LocalDirAllocator lDirAlloc = - new LocalDirAllocator(TezRuntimeFrameworkConfigs.LOCAL_DIRS); + new LocalDirAllocator(TezRuntimeFrameworkConfigs.LOCAL_DIRS); /* * if service_id = mapreduce_shuffle then "${appDir}/output/${uniqueId}" @@ -70,7 +70,6 @@ private Path getAttemptOutputDir() { return new Path(dagPath, uniqueId); } - /** * Create a local output file name. * @@ -86,7 +85,7 @@ private Path getAttemptOutputDir() { @Override public Path getOutputFileForWrite(long size) throws IOException { Path attemptOutput = - new Path(getAttemptOutputDir(), Constants.TEZ_RUNTIME_TASK_OUTPUT_FILENAME_STRING); + new Path(getAttemptOutputDir(), Constants.TEZ_RUNTIME_TASK_OUTPUT_FILENAME_STRING); return lDirAlloc.getLocalPathForWrite(attemptOutput.toString(), size, conf); } @@ -106,7 +105,7 @@ public Path getOutputFileForWrite(long size) throws IOException { @Override public Path getOutputFileForWrite() throws IOException { Path attemptOutput = - new Path(getAttemptOutputDir(), Constants.TEZ_RUNTIME_TASK_OUTPUT_FILENAME_STRING); + new Path(getAttemptOutputDir(), Constants.TEZ_RUNTIME_TASK_OUTPUT_FILENAME_STRING); return lDirAlloc.getLocalPathForWrite(attemptOutput.toString(), conf); } @@ -136,7 +135,6 @@ public Path getOutputFileForWriteInVolume(Path existing) { return new Path(attemptDir, Constants.TEZ_RUNTIME_TASK_OUTPUT_FILENAME_STRING); } - /** * Create a local output index file name. * @@ -152,8 +150,8 @@ public Path getOutputFileForWriteInVolume(Path existing) { @Override public Path getOutputIndexFileForWrite(long size) throws IOException { Path attemptIndexOutput = - new Path(getAttemptOutputDir(), Constants.TEZ_RUNTIME_TASK_OUTPUT_FILENAME_STRING + - Constants.TEZ_RUNTIME_TASK_OUTPUT_INDEX_SUFFIX_STRING); + new Path(getAttemptOutputDir(), Constants.TEZ_RUNTIME_TASK_OUTPUT_FILENAME_STRING + + Constants.TEZ_RUNTIME_TASK_OUTPUT_INDEX_SUFFIX_STRING); return lDirAlloc.getLocalPathForWrite(attemptIndexOutput.toString(), size, conf); } @@ -230,7 +228,6 @@ public Path getSpillIndexFileForWrite(int spillNumber, long size) return lDirAlloc.getLocalPathForWrite(outputDir.toString(), size, conf); } - /** * Create a local input file name. * @@ -247,7 +244,7 @@ public Path getSpillIndexFileForWrite(int spillNumber, long size) */ @Override public Path getInputFileForWrite(int srcIdentifier, - int spillNum, long size) throws IOException { + int spillNum, long size) throws IOException { String dagPath = getDagOutputDir(getSpillFileName(srcIdentifier, spillNum)); return lDirAlloc.getLocalPathForWrite(dagPath, size, conf); } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/package-info.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/package-info.java index 14f7fcc4fb..22b620083a 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/package-info.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/task/local/output/package-info.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,4 +19,4 @@ @Private package org.apache.tez.runtime.library.common.task.local.output; -import org.apache.hadoop.classification.InterfaceAudience.Private; \ No newline at end of file +import org.apache.hadoop.classification.InterfaceAudience.Private; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/BaseUnorderedPartitionedKVWriter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/BaseUnorderedPartitionedKVWriter.java index adea49fe80..b509dfaadb 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/BaseUnorderedPartitionedKVWriter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/BaseUnorderedPartitionedKVWriter.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.runtime.library.common.writers; @@ -47,7 +47,7 @@ public abstract class BaseUnorderedPartitionedKVWriter extends KeyValuesWriter { private static final Logger LOG = LoggerFactory.getLogger(BaseUnorderedPartitionedKVWriter.class); - + protected final OutputContext outputContext; protected final Configuration conf; protected final RawLocalFileSystem localFs; @@ -62,7 +62,7 @@ public abstract class BaseUnorderedPartitionedKVWriter extends KeyValuesWriter { protected final int numPartitions; protected final CompressionCodec codec; protected final TezTaskOutput outputFileHandler; - + protected final boolean ifileReadAhead; protected final int ifileReadAheadLength; protected final int ifileBufferSize; @@ -82,7 +82,7 @@ public abstract class BaseUnorderedPartitionedKVWriter extends KeyValuesWriter { * meta-information. */ protected final TezCounter outputBytesWithOverheadCounter; - + /** * Represents the final output size, with file format overheads and compression factored in. * Does not consider spills. @@ -121,7 +121,7 @@ public BaseUnorderedPartitionedKVWriter(OutputContext outputContext, Configurati throw new RuntimeException(e); } this.numPartitions = numOutputs; - + // k/v serialization keyClass = ConfigUtils.getIntermediateOutputKeyClass(this.conf); valClass = ConfigUtils.getIntermediateOutputValueClass(this.conf); @@ -130,7 +130,7 @@ public BaseUnorderedPartitionedKVWriter(OutputContext outputContext, Configurati valSerialization = serializationFactory.getSerialization(valClass); keySerializer = keySerialization.getSerializer(keyClass); valSerializer = valSerialization.getSerializer(valClass); - + outputRecordBytesCounter = outputContext.getCounters().findCounter(TaskCounter.OUTPUT_BYTES); outputRecordsCounter = outputContext.getCounters().findCounter(TaskCounter.OUTPUT_RECORDS); outputBytesWithOverheadCounter = outputContext.getCounters().findCounter(TaskCounter.OUTPUT_BYTES_WITH_OVERHEAD); @@ -160,7 +160,7 @@ public BaseUnorderedPartitionedKVWriter(OutputContext outputContext, Configurati } this.ifileBufferSize = conf.getInt("io.file.buffer.size", TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_BUFFER_SIZE_DEFAULT); - + LOG.info("Instantiating Partitioner: [" + conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS) + "]"); try { this.partitioner = TezRuntimeUtils.instantiatePartitioner(this.conf); @@ -177,11 +177,10 @@ public BaseUnorderedPartitionedKVWriter(OutputContext outputContext, Configurati public void write(Object key, Iterable values) throws IOException { //TODO: UnorderedPartitionedKVWriter should override this method later. Iterator it = values.iterator(); - while(it.hasNext()) { + while (it.hasNext()) { write(key, it.next()); } } public abstract List close() throws IOException, InterruptedException; - } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java index faf75866b9..311368ce3e 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -201,7 +201,7 @@ public Deflater get() { private final boolean useCachedStream; public UnorderedPartitionedKVWriter(OutputContext outputContext, Configuration conf, - int numOutputs, long availableMemoryBytes) throws IOException { + int numOutputs, long availableMemoryBytes) throws IOException { super(outputContext, conf, numOutputs); Preconditions.checkArgument(availableMemoryBytes >= 0, "availableMemory should be >= 0 bytes"); @@ -220,13 +220,13 @@ public UnorderedPartitionedKVWriter(OutputContext outputContext, Configuration c this.finalEvents = Lists.newLinkedList(); this.dataViaEventsEnabled = conf.getBoolean( - TezRuntimeConfiguration.TEZ_RUNTIME_TRANSFER_DATA_VIA_EVENTS_ENABLED, - TezRuntimeConfiguration.TEZ_RUNTIME_TRANSFER_DATA_VIA_EVENTS_ENABLED_DEFAULT); + TezRuntimeConfiguration.TEZ_RUNTIME_TRANSFER_DATA_VIA_EVENTS_ENABLED, + TezRuntimeConfiguration.TEZ_RUNTIME_TRANSFER_DATA_VIA_EVENTS_ENABLED_DEFAULT); // No max cap on size (intentional) this.dataViaEventsMaxSize = conf.getInt( - TezRuntimeConfiguration.TEZ_RUNTIME_TRANSFER_DATA_VIA_EVENTS_MAX_SIZE, - TezRuntimeConfiguration.TEZ_RUNTIME_TRANSFER_DATA_VIA_EVENTS_MAX_SIZE_DEFAULT); + TezRuntimeConfiguration.TEZ_RUNTIME_TRANSFER_DATA_VIA_EVENTS_MAX_SIZE, + TezRuntimeConfiguration.TEZ_RUNTIME_TRANSFER_DATA_VIA_EVENTS_MAX_SIZE_DEFAULT); boolean useCachedStreamConfig = conf.getBoolean( TezRuntimeConfiguration.TEZ_RUNTIME_TRANSFER_DATA_VIA_EVENTS_SUPPORT_IN_MEM_FILE, @@ -268,7 +268,7 @@ public UnorderedPartitionedKVWriter(OutputContext outputContext, Configuration c valSerializer.open(dos); rfs = ((LocalFileSystem) FileSystem.getLocal(this.conf)).getRaw(); - int maxThreads = Math.max(2, numBuffers/2); + int maxThreads = Math.max(2, numBuffers / 2); //TODO: Make use of TezSharedExecutor later ExecutorService executor = new ThreadPoolExecutor(1, maxThreads, 60L, TimeUnit.SECONDS, @@ -288,7 +288,7 @@ public UnorderedPartitionedKVWriter(OutputContext outputContext, Configuration c numRecordsPerPartition = new int[numPartitions]; reportPartitionStats = ReportPartitionStats.fromString( conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_REPORT_PARTITION_STATS, - TezRuntimeConfiguration.TEZ_RUNTIME_REPORT_PARTITION_STATS_DEFAULT)); + TezRuntimeConfiguration.TEZ_RUNTIME_REPORT_PARTITION_STATS_DEFAULT)); sizePerPartition = (reportPartitionStats.isEnabled()) ? new long[numPartitions] : null; @@ -333,18 +333,19 @@ public UnorderedPartitionedKVWriter(OutputContext outputContext, Configuration c } private static final int ALLOC_OVERHEAD = 64; + private void computeNumBuffersAndSize(int bufferLimit) { - numBuffers = (int)(availableMemory / bufferLimit); + numBuffers = (int) (availableMemory / bufferLimit); if (numBuffers >= 2) { sizePerBuffer = bufferLimit - ALLOC_OVERHEAD; - lastBufferSize = (int)(availableMemory % bufferLimit); + lastBufferSize = (int) (availableMemory % bufferLimit); // Use leftover memory last buffer only if the leftover memory > 50% of bufferLimit if (lastBufferSize > bufferLimit / 2) { numBuffers += 1; } else { if (lastBufferSize > 0) { - LOG.warn("Underallocating memory. Unused memory size: {}.", lastBufferSize); + LOG.warn("Underallocating memory. Unused memory size: {}.", lastBufferSize); } lastBufferSize = sizePerBuffer; } @@ -354,7 +355,7 @@ private void computeNumBuffersAndSize(int bufferLimit) { if (availableMemory / numBuffers > Integer.MAX_VALUE) { sizePerBuffer = Integer.MAX_VALUE; } else { - sizePerBuffer = (int)(availableMemory / numBuffers); + sizePerBuffer = (int) (availableMemory / numBuffers); } // 2 equal sized buffers. lastBufferSize = sizePerBuffer; @@ -473,7 +474,6 @@ private void write(Object key, Object value, int partition) throws IOException { currentBuffer.sizePerPartition[partition] += currentBuffer.nextPosition - (metaStart + META_SIZE); currentBuffer.numRecords++; - } private void updateTezCountersAndNotify() { @@ -545,7 +545,7 @@ private boolean scheduleSpill(boolean block) throws IOException { // Update once per buffer (instead of every record) updateTezCountersAndNotify(); return true; - } catch(InterruptedException ie) { + } catch (InterruptedException ie) { Thread.currentThread().interrupt(); // reset interrupt status } return false; @@ -598,7 +598,7 @@ private class SpillCallable extends CallableWithNdc { private int spillNumber; public SpillCallable(List filledBuffers, CompressionCodec codec, - TezCounter numRecordsCounter, SpillPathDetails spillPathDetails) { + TezCounter numRecordsCounter, SpillPathDetails spillPathDetails) { this(filledBuffers, codec, numRecordsCounter, spillPathDetails.spillIndex); Preconditions.checkArgument(spillPathDetails.outputFilePath != null, "Spill output file " + "path can not be null"); @@ -606,7 +606,7 @@ public SpillCallable(List filledBuffers, CompressionCodec codec, } public SpillCallable(List filledBuffers, CompressionCodec codec, - TezCounter numRecordsCounter, int spillNumber) { + TezCounter numRecordsCounter, int spillNumber) { this.filledBuffers = filledBuffers; this.codec = codec; this.numRecordsCounter = numRecordsCounter; @@ -685,7 +685,7 @@ protected SpillResult callInternal() throws IOException { } private long writePartition(int pos, WrappedBuffer wrappedBuffer, Writer writer, - DataInputBuffer keyBuffer, DataInputBuffer valBuffer) throws IOException { + DataInputBuffer keyBuffer, DataInputBuffer valBuffer) throws IOException { long numRecords = 0; while (pos != WrappedBuffer.PARTITION_ABSENT_POSITION) { int metaIndex = pos / INT_SIZE; @@ -730,7 +730,7 @@ private boolean canSendDataOverDME() throws IOException { } return (writer != null) && (dataViaEventsEnabled) - && (writer.getCompressedLength() <= dataViaEventsMaxSize); + && (writer.getCompressedLength() <= dataViaEventsMaxSize); } private ByteBuffer readDataForDME() throws IOException { @@ -811,7 +811,7 @@ public List close() throws IOException, InterruptedException { sr.writeToFile(finalIndexPath, conf, localFs); } eventList.add(generateDMEvent(false, -1, false, outputContext - .getUniqueIdentifier(), emptyPartitions)); + .getUniqueIdentifier(), emptyPartitions)); return eventList; } @@ -875,7 +875,7 @@ private BitSet getEmptyPartitions(int[] recordsPerPartition) { Preconditions.checkArgument(recordsPerPartition != null, "records per partition can not be null"); BitSet emptyPartitions = new BitSet(); for (int i = 0; i < numPartitions; i++) { - if (recordsPerPartition[i] == 0 ) { + if (recordsPerPartition[i] == 0) { emptyPartitions.set(i); } } @@ -897,7 +897,7 @@ private Event generateDMEvent() throws IOException { } private Event generateDMEvent(boolean addSpillDetails, int spillId, - boolean isLastSpill, String pathComponent, BitSet emptyPartitions) + boolean isLastSpill, String pathComponent, BitSet emptyPartitions) throws IOException { outputContext.notifyProgress(); @@ -998,10 +998,9 @@ private SpillResult finalSpill() throws IOException { fileOutputBytesCounter.increment(indexFileSizeEstimate); return spillResult; } catch (Exception ex) { - throw (ex instanceof IOException) ? (IOException)ex : new IOException(ex); + throw (ex instanceof IOException) ? (IOException) ex : new IOException(ex); } } - } /** @@ -1028,7 +1027,7 @@ private SpillPathDetails getSpillPathDetails(boolean isFinalSpill, long expected * @throws IOException */ private SpillPathDetails getSpillPathDetails(boolean isFinalSpill, long expectedSpillSize, - int spillNumber) throws IOException { + int spillNumber) throws IOException { long spillSize = (expectedSpillSize < 0) ? (currentBuffer.nextPosition + numPartitions * APPROX_HEADER_LENGTH) : expectedSpillSize; @@ -1048,7 +1047,7 @@ private SpillPathDetails getSpillPathDetails(boolean isFinalSpill, long expected } } else { outputFilePath = outputFileHandler.getSpillFileForWrite(spillNumber, spillSize); - indexFilePath = outputFileHandler.getSpillIndexFileForWrite(spillNumber, indexFileSizeEstimate); + indexFilePath = outputFileHandler.getSpillIndexFileForWrite(spillNumber, indexFileSizeEstimate); } return new SpillPathDetails(outputFilePath, indexFilePath, spillNumber); @@ -1250,7 +1249,7 @@ public void write(int v) throws IOException { public void write(byte[] b, int off, int len) throws IOException { if (currentBuffer.full) { - /* no longer do anything until reset */ + /* no longer do anything until reset */ } else if (len > currentBuffer.availableSize) { currentBuffer.full = true; /* stop working & signal we hit the end */ } else { @@ -1323,8 +1322,8 @@ private String generatePathComponent(String uniqueId, int spillNumber) { } private List generateEventForSpill(BitSet emptyPartitions, long[] sizePerPartition, - int spillNumber, - boolean isFinalUpdate) throws IOException { + int spillNumber, + boolean isFinalUpdate) throws IOException { List eventList = Lists.newLinkedList(); //Send out an event for consuming. String pathComponent = generatePathComponent(outputContext.getUniqueIdentifier(), spillNumber); @@ -1366,7 +1365,7 @@ private void mayBeSendEventsForSpill( } private void mayBeSendEventsForSpill(int[] recordsPerPartition, - long[] sizePerPartition, int spillNumber, boolean isFinalUpdate) { + long[] sizePerPartition, int spillNumber, boolean isFinalUpdate) { BitSet emptyPartitions = getEmptyPartitions(recordsPerPartition); mayBeSendEventsForSpill(emptyPartitions, sizePerPartition, spillNumber, isFinalUpdate); @@ -1423,11 +1422,11 @@ public void onSuccess(SpillResult result) { } if (!pipelinedShuffle && isFinalMergeEnabled) { - synchronized(additionalSpillBytesWritternCounter) { + synchronized (additionalSpillBytesWritternCounter) { additionalSpillBytesWritternCounter.increment(result.spillSize); } } else { - synchronized(fileOutputBytesCounter) { + synchronized (fileOutputBytesCounter) { fileOutputBytesCounter.increment(indexFileSizeEstimate); fileOutputBytesCounter.increment(result.spillSize); } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/package-info.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/package-info.java index 2376c99e66..729450ed02 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/package-info.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/package-info.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,4 +19,4 @@ @Private package org.apache.tez.runtime.library.common.writers; -import org.apache.hadoop.classification.InterfaceAudience.Private; \ No newline at end of file +import org.apache.hadoop.classification.InterfaceAudience.Private; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/HadoopKeyValuesBasedBaseEdgeConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/HadoopKeyValuesBasedBaseEdgeConfig.java index 0692bac61f..b5c52d6a15 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/HadoopKeyValuesBasedBaseEdgeConfig.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/HadoopKeyValuesBasedBaseEdgeConfig.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -46,7 +46,6 @@ abstract class HadoopKeyValuesBasedBaseEdgeConfig { */ public abstract UserPayload getInputPayload(); - /** * Get the history text for the configured Output * @return output configuration as a string in json format @@ -79,8 +78,6 @@ public abstract static class Builder> implements BaseConfig * @return instance of the current builder */ public abstract T setCompression(boolean enabled, @Nullable String compressionCodec, - @Nullable Map codecConf); - + @Nullable Map codecConf); } - } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedGroupedKVInputConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedGroupedKVInputConfig.java index d9c1d21403..4dc25cd636 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedGroupedKVInputConfig.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedGroupedKVInputConfig.java @@ -58,9 +58,11 @@ public static interface SpecificConfigBuilder extends BaseConfigBuilder { /** * Specifies whether the legacy version of this input should be used. + * * @return instance of the current builder */ public T useLegacyInput(); + /** * Sets the buffer fraction, as a fraction of container size, to be used while fetching remote * data. @@ -128,7 +130,6 @@ public static interface SpecificConfigBuilder extends BaseConfigBuilder { * @return instance of the current builder */ public T setCombiner(String combinerClassName, @Nullable Map combinerConf); - } @SuppressWarnings("rawtypes") @@ -140,7 +141,6 @@ public static class SpecificBuilder setCombiner(String combinerClassName, Map setAdditionalConfiguration(String key, String value) { builder.setAdditionalConfiguration(key, value); @@ -222,7 +221,6 @@ public SpecificBuilder setFromConfigurationUnfiltered( public E done() { return edgeBuilder; } - } @InterfaceAudience.Private @@ -247,6 +245,7 @@ private OrderedGroupedKVInputConfig(Configuration conf, boolean useLegacyInput) /** * Get a UserPayload representation of the Configuration + * * @return a {@link org.apache.tez.dag.api.UserPayload} instance */ public UserPayload toUserPayload() { @@ -293,8 +292,8 @@ public static class Builder implements SpecificConfigBuilder { /** * Create a configuration builder for {@link org.apache.tez.runtime.library.input.OrderedGroupedKVInput} * - * @param keyClassName the key class name - * @param valueClassName the value class name + * @param keyClassName the key class name + * @param valueClassName the value class name */ @InterfaceAudience.Private Builder(String keyClassName, String valueClassName) { @@ -492,7 +491,7 @@ public Builder setCompression(boolean enabled, @Nullable String compressionCodec * @return this object for further chained method calls */ public Builder setKeySerializationClass(String serializationClassName, - String comparatorClassName, @Nullable Map serializerConf) { + String comparatorClassName, @Nullable Map serializerConf) { Objects.requireNonNull(serializationClassName, "serializationClassName cannot be null"); Objects.requireNonNull(comparatorClassName, "comparator cannot be null"); this.conf.set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, serializationClassName + "," @@ -537,5 +536,4 @@ public OrderedGroupedKVInputConfig build() { return new OrderedGroupedKVInputConfig(this.conf, this.useLegacyInput); } } - } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java index e4e9be2de0..0c95e415b9 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -224,7 +224,7 @@ public Builder setKeyComparatorClass(String comparatorClassName, * @return this object for further chained method calls */ public Builder setKeySerializationClass(String serializationClassName, - String comparatorClassName, @Nullable Map serializerConf) { + String comparatorClassName, @Nullable Map serializerConf) { outputBuilder.setKeySerializationClass(serializationClassName, comparatorClassName, serializerConf); inputBuilder.setKeySerializationClass(serializationClassName, comparatorClassName, serializerConf); return this; @@ -246,7 +246,6 @@ public Builder setValueSerializationClass(String serializationClassName, return this; } - @Override public Builder setCompression(boolean enabled, @Nullable String compressionCodec, @Nullable Map codecConf) { @@ -313,6 +312,5 @@ public OrderedGroupedKVInputConfig.SpecificBuilder configureInput() { public OrderedPartitionedKVEdgeConfig build() { return new OrderedPartitionedKVEdgeConfig(outputBuilder.build(), inputBuilder.build()); } - } } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java index 9671feb8b9..922f6a62ee 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java @@ -39,12 +39,11 @@ import org.apache.tez.runtime.library.common.ConfigUtils; import org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput; - @InterfaceAudience.Public @InterfaceStability.Evolving /** * Configure {@link org.apache.tez.runtime.library.output.OrderedPartitionedKVOutput}

    - * + * * Values will be picked up from tez-site if not specified, otherwise defaults from * {@link org.apache.tez.runtime.library.api.TezRuntimeConfiguration} will be used. */ @@ -54,15 +53,17 @@ public class OrderedPartitionedKVOutputConfig { * Currently supported sorter implementations */ public enum SorterImpl { - /** Legacy sorter implementation based on Hadoop MapReduce shuffle impl. + /** + * Legacy sorter implementation based on Hadoop MapReduce shuffle impl. * Restricted to 2 GB memory limits. */ LEGACY, - /** Pipeline sorter - a more efficient sorter that supports > 2 GB sort buffers */ + /** + * Pipeline sorter - a more efficient sorter that supports > 2 GB sort buffers + */ PIPELINED } - /** * Configure parameters which are specific to the Output. */ @@ -76,7 +77,6 @@ public static interface SpecificConfigBuilder extends BaseConfigBuilder { */ public T setSortBufferSize(int sortBufferSize); - /** * Configure the combiner class * @@ -113,7 +113,6 @@ public static interface SpecificConfigBuilder extends BaseConfigBuilder { * @return instance of the current builder */ public T setSorter(SorterImpl sorterImpl); - } @SuppressWarnings("rawtypes") @@ -158,7 +157,6 @@ public SpecificBuilder setSorter(SorterImpl sorterImpl) { return this; } - @Override public SpecificBuilder setAdditionalConfiguration(String key, String value) { builder.setAdditionalConfiguration(key, value); @@ -204,6 +202,7 @@ private OrderedPartitionedKVOutputConfig(Configuration conf) { /** * Get a UserPayload representation of the Configuration + * * @return a {@link org.apache.tez.dag.api.UserPayload} instance */ public UserPayload toUserPayload() { @@ -259,7 +258,7 @@ public static class Builder implements SpecificConfigBuilder { */ @InterfaceAudience.Private Builder(String keyClassName, String valueClassName, String partitionerClassName, - @Nullable Map partitionerConf) { + @Nullable Map partitionerConf) { this(); Objects.requireNonNull(keyClassName, "Key class name cannot be null"); Objects.requireNonNull(valueClassName, "Value class name cannot be null"); @@ -340,7 +339,6 @@ public Builder setSorter(SorterImpl sorterImpl) { return this; } - @SuppressWarnings("unchecked") @Override public Builder setAdditionalConfiguration(String key, String value) { @@ -453,10 +451,10 @@ public Builder setCompression(boolean enabled, @Nullable String compressionCodec * to the ones required by the comparator. * @return this object for further chained method calls * @throws NullPointerException if {@code serializationClassName} or - * {@code comparatorClassName} is {@code null} + * {@code comparatorClassName} is {@code null} */ public Builder setKeySerializationClass(String serializationClassName, - String comparatorClassName, @Nullable Map serializerConf) { + String comparatorClassName, @Nullable Map serializerConf) { Objects.requireNonNull(serializationClassName, "serializationClassName cannot be null"); Objects.requireNonNull(comparatorClassName, "comparator cannot be null"); this.conf.set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, serializationClassName + "," @@ -503,4 +501,3 @@ public OrderedPartitionedKVOutputConfig build() { } } } - diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java index c4e8694a8a..4cc3604e9a 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java @@ -38,7 +38,7 @@ /** * Configure payloads for the UnorderedKVOutput and UnorderedKVInput pair

    - * + *

    * Values will be picked up from tez-site if not specified, otherwise defaults from * {@link org.apache.tez.runtime.library.api.TezRuntimeConfiguration} will be used. */ @@ -53,12 +53,12 @@ private UnorderedKVEdgeConfig( UnorderedKVInputConfig inputConfiguration) { this.outputConf = outputConfiguration; this.inputConf = inputConfiguration; - } /** * Create a builder to configure the relevant Input and Output - * @param keyClassName the key class name + * + * @param keyClassName the key class name * @param valueClassName the value class name * @return a builder to configure the edge */ @@ -260,6 +260,7 @@ public Builder setValueSerializationClass(String serializationClassName, /** * Configure the specific output + * * @return a builder to configure the output */ public UnorderedKVOutputConfig.SpecificBuilder configureOutput() { @@ -268,6 +269,7 @@ public UnorderedKVOutputConfig.SpecificBuilder configureOutput() { /** * Configure the specific input + * * @return a builder to configure the input */ public UnorderedKVInputConfig.SpecificBuilder configureInput() { @@ -276,11 +278,11 @@ public UnorderedKVInputConfig.SpecificBuilder configureInput() { /** * Build and return an instance of the configuration + * * @return an instance of the acatual configuration */ public UnorderedKVEdgeConfig build() { return new UnorderedKVEdgeConfig(outputBuilder.build(), inputBuilder.build()); } - } } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVInputConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVInputConfig.java index 6f285e7995..cc375131ae 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVInputConfig.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVInputConfig.java @@ -84,7 +84,6 @@ public static interface SpecificConfigBuilder extends BaseConfigBuilder { * @return instance of the current builder */ public T setMergeFraction(float mergeFraction); - } @SuppressWarnings("rawtypes") @@ -96,7 +95,6 @@ public static class SpecificBuilder { /** * Create a configuration builder for {@link org.apache.tez.runtime.library.input.UnorderedKVInput} * - * @param keyClassName the key class name - * @param valueClassName the value class name + * @param keyClassName the key class name + * @param valueClassName the value class name */ @InterfaceAudience.Private Builder(String keyClassName, String valueClassName) { diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVOutputConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVOutputConfig.java index a6ac7f2b5a..8538967b86 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVOutputConfig.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVOutputConfig.java @@ -113,6 +113,7 @@ private UnorderedKVOutputConfig(Configuration conf) { /** * Get a UserPayload representation of the Configuration + * * @return a {@link org.apache.tez.dag.api.UserPayload} instance */ public UserPayload toUserPayload() { @@ -154,8 +155,8 @@ public static class Builder implements SpecificConfigBuilder { /** * Create a configuration builder for {@link org.apache.tez.runtime.library.output.UnorderedKVOutput} * - * @param keyClassName the key class name - * @param valueClassName the value class name + * @param keyClassName the key class name + * @param valueClassName the value class name */ @InterfaceAudience.Private Builder(String keyClassName, String valueClassName) { diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java index 0d8a5aef75..e084bfec53 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java @@ -38,7 +38,7 @@ /** * Configure payloads for the UnorderedPartitionedKVOutput and UnorderedKVInput pair

    - * + *

    * Values will be picked up from tez-site if not specified, otherwise defaults from * {@link org.apache.tez.runtime.library.api.TezRuntimeConfiguration} will be used. */ @@ -55,7 +55,6 @@ private UnorderedPartitionedKVEdgeConfig( UnorderedKVInputConfig inputConfiguration) { this.outputConf = outputConfiguration; this.inputConf = inputConfiguration; - } /** @@ -273,6 +272,7 @@ public UnorderedPartitionedKVOutputConfig.SpecificBuilder configureOutp /** * Configure the specific input + * * @return a builder to configure the input */ public UnorderedKVInputConfig.SpecificBuilder configureInput() { @@ -281,11 +281,11 @@ public UnorderedKVInputConfig.SpecificBuilder configureInput() { /** * Build and return an instance of the configuration + * * @return an instance of the acatual configuration */ public UnorderedPartitionedKVEdgeConfig build() { return new UnorderedPartitionedKVEdgeConfig(outputBuilder.build(), inputBuilder.build()); } - } } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVOutputConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVOutputConfig.java index dbbfb9ea6d..757b65447d 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVOutputConfig.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVOutputConfig.java @@ -127,6 +127,7 @@ private UnorderedPartitionedKVOutputConfig(Configuration conf) { /** * Get a UserPayload representation of the Configuration + * * @return a {@link org.apache.tez.dag.api.UserPayload} instance */ public UserPayload toUserPayload() { @@ -181,7 +182,7 @@ public static class Builder implements SpecificConfigBuilder { */ @InterfaceAudience.Private Builder(String keyClassName, String valueClassName, String partitionerClassName, - Map partitionerConf) { + Map partitionerConf) { this(); Objects.requireNonNull(keyClassName, "Key class name cannot be null"); Objects.requireNonNull(valueClassName, "Value class name cannot be null"); @@ -315,7 +316,8 @@ public Builder setKeySerializationClass(String serializationClassName, // Merging the confs for now. Change to be specific in the future. ConfigUtils.mergeConfsWithExclusions(this.conf, serializerConf, TezRuntimeConfiguration.getRuntimeConfigKeySet()); - } return this; + } + return this; } /** diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/Utils.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/Utils.java index c1b44a29d3..df797f1e61 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/Utils.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/Utils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -30,7 +30,7 @@ class Utils { * @param edgeProperty Edge property to be be modified */ static void setEdgePropertyHistoryText(HadoopKeyValuesBasedBaseEdgeConfig edgeConfig, - EdgeProperty edgeProperty) { + EdgeProperty edgeProperty) { String inputHistoryText = edgeConfig.getInputHistoryText(); if (inputHistoryText != null) { edgeProperty.getEdgeDestination().setHistoryText(inputHistoryText); @@ -40,6 +40,4 @@ static void setEdgePropertyHistoryText(HadoopKeyValuesBasedBaseEdgeConfig edgeCo edgeProperty.getEdgeSource().setHistoryText(outputHistoryText); } } - - } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/package-info.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/package-info.java index 16dd5fff34..938ec1ed5e 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/package-info.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/package-info.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,4 +21,4 @@ package org.apache.tez.runtime.library.conf; import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Evolving; \ No newline at end of file +import org.apache.hadoop.classification.InterfaceStability.Evolving; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/exceptions/FetcherReadTimeoutException.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/exceptions/FetcherReadTimeoutException.java index 32b31874c5..64bed41905 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/exceptions/FetcherReadTimeoutException.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/exceptions/FetcherReadTimeoutException.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,7 +18,6 @@ package org.apache.tez.runtime.library.exceptions; - import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.tez.dag.api.TezException; @@ -45,5 +44,4 @@ public FetcherReadTimeoutException(Throwable cause) { public FetcherReadTimeoutException(String message, Throwable cause) { super(message, cause); } - } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/exceptions/InputAlreadyClosedException.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/exceptions/InputAlreadyClosedException.java index 91b1a13dc6..700e6f4bdd 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/exceptions/InputAlreadyClosedException.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/exceptions/InputAlreadyClosedException.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,7 +18,6 @@ package org.apache.tez.runtime.library.exceptions; - import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.tez.dag.api.TezException; @@ -43,5 +42,4 @@ public InputAlreadyClosedException(Throwable cause) { public InputAlreadyClosedException(String message, Throwable cause) { super(message, cause); } - } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/hadoop/compat/NullProgressable.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/hadoop/compat/NullProgressable.java index c370696d95..4fbafe7bc0 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/hadoop/compat/NullProgressable.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/hadoop/compat/NullProgressable.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.runtime.library.hadoop.compat; @@ -31,5 +31,4 @@ public NullProgressable() { @Override public void progress() { } - } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValueInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValueInput.java index a0059cfbb3..eb03911e55 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValueInput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValueInput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -91,7 +91,7 @@ public Object getCurrentValue() throws IOException { } public float getProgress() throws IOException, InterruptedException { - return (1.0f)*(currentReaderIndex + 1)/getInputs().size(); + return (1.0f) * (currentReaderIndex + 1) / getInputs().size(); } } @@ -118,4 +118,4 @@ public float getProgress() throws ProgressFailedException, InterruptedException throw new ProgressFailedException("getProgress encountered IOException ", e); } } -} \ No newline at end of file +} diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValuesInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValuesInput.java index 2555a5724d..5c2e952585 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValuesInput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/ConcatenatedMergedKeyValuesInput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -73,7 +73,7 @@ public boolean next() throws IOException { // An InterruptedException is not expected here since this works off of // underlying readers which take care of throwing IOInterruptedExceptions if (e instanceof IOException) { - throw (IOException)e; + throw (IOException) e; } else { throw new IOException(e); } @@ -93,10 +93,10 @@ public Iterable getCurrentValues() throws IOException { } public float getProgress() throws IOException, InterruptedException { - return (1.0f)*(currentReaderIndex + 1)/getInputs().size(); + return (1.0f) * (currentReaderIndex + 1) / getInputs().size(); } } - + /** * Provides a {@link KeyValuesReader} that iterates over the * concatenated input data @@ -120,4 +120,4 @@ public float getProgress() throws ProgressFailedException, InterruptedException throw new ProgressFailedException("getProgress encountered IOException ", e); } } -} \ No newline at end of file +} diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedInputLegacy.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedInputLegacy.java index b697be5a1d..3e999994d3 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedInputLegacy.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedInputLegacy.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -39,49 +39,49 @@ public OrderedGroupedInputLegacy(InputContext inputContext, int numPhysicalInput @Private public TezRawKeyValueIterator getIterator() throws IOException, InterruptedException, TezException { // wait for input so that iterator is available - synchronized(this) { - if (getNumPhysicalInputs() == 0) { - return new TezRawKeyValueIterator() { - @Override - public DataInputBuffer getKey() throws IOException { - throw new RuntimeException("No data available in Input"); - } + synchronized (this) { + if (getNumPhysicalInputs() == 0) { + return new TezRawKeyValueIterator() { + @Override + public DataInputBuffer getKey() throws IOException { + throw new RuntimeException("No data available in Input"); + } - @Override - public DataInputBuffer getValue() throws IOException { - throw new RuntimeException("No data available in Input"); - } + @Override + public DataInputBuffer getValue() throws IOException { + throw new RuntimeException("No data available in Input"); + } - @Override - public boolean next() throws IOException { - return false; - } + @Override + public boolean next() throws IOException { + return false; + } - @Override - public boolean hasNext() throws IOException { - return false; - } + @Override + public boolean hasNext() throws IOException { + return false; + } - @Override - public void close() throws IOException { - } + @Override + public void close() throws IOException { + } - @Override - public Progress getProgress() { - progress.complete(); - return progress; - } + @Override + public Progress getProgress() { + progress.complete(); + return progress; + } - @Override - public boolean isSameKey() throws IOException { - throw new UnsupportedOperationException("isSameKey is not supported"); - } - }; - } + @Override + public boolean isSameKey() throws IOException { + throw new UnsupportedOperationException("isSameKey is not supported"); + } + }; + } } waitForInputReady(); - synchronized(this) { + synchronized (this) { return rawIter; } } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedKVInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedKVInput.java index 1463cfabbe..099b2fd5d9 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedKVInput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedKVInput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -56,7 +56,6 @@ import org.apache.tez.common.Preconditions; - /** * {@link OrderedGroupedKVInput} in a {@link AbstractLogicalInput} which shuffles * intermediate sorted data, merges them and provides key/ to the @@ -116,7 +115,7 @@ public synchronized List initialize() throws IOException { this.inputKeyCounter = getContext().getCounters().findCounter(TaskCounter.REDUCE_INPUT_GROUPS); this.inputValueCounter = getContext().getCounters().findCounter( TaskCounter.REDUCE_INPUT_RECORDS); - this.shuffledInputs = getContext().getCounters().findCounter( + this.shuffledInputs = getContext().getCounters().findCounter( TaskCounter.NUM_SHUFFLED_INPUTS); this.conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, getContext().getWorkDirs()); @@ -183,7 +182,7 @@ public void waitForInputReady() throws IOException, InterruptedException, TezExc } TezRawKeyValueIterator localRawIter = localShuffleCopy.waitForInput(); - synchronized(this) { + synchronized (this) { rawIter = localRawIter; createValuesIterator(); } @@ -197,14 +196,14 @@ public synchronized List close() throws IOException { if (shuffle != null) { shuffle.shutdown(); } - + long dataSize = getContext().getCounters() .findCounter(TaskCounter.SHUFFLE_BYTES_DECOMPRESSED).getValue(); getContext().getStatisticsReporter().reportDataSize(dataSize); long inputRecords = getContext().getCounters() .findCounter(TaskCounter.REDUCE_INPUT_RECORDS).getValue(); getContext().getStatisticsReporter().reportItemsProcessed(inputRecords); - + return Collections.emptyList(); } @@ -259,7 +258,7 @@ public Iterable getCurrentValues() throws IOException { } @SuppressWarnings("rawtypes") ValuesIterator valuesIter = null; - synchronized(this) { + synchronized (this) { valuesIter = vIter; } return new OrderedGroupedKeyValuesReader(valuesIter, getContext()); @@ -272,7 +271,7 @@ public float getProgress() throws ProgressFailedException, InterruptedException synchronized (this) { return ((0.5f) * this.shuffledInputs.getValue() / totalInputs) + ((rawIter != null) ? - ((0.5f) * rawIter.getProgress().getProgress()) : 0.0f); + ((0.5f) * rawIter.getProgress().getProgress()) : 0.0f); } } else { return 0.0f; @@ -298,7 +297,7 @@ public void handleEvents(List inputEvents) throws IOException { shuffleLocalRef.handleEvents(inputEvents); } - @SuppressWarnings({ "rawtypes", "unchecked" }) + @SuppressWarnings({"rawtypes", "unchecked"}) protected synchronized void createValuesIterator() throws IOException { // Not used by ReduceProcessor @@ -312,7 +311,6 @@ protected synchronized void createValuesIterator() vIter = new ValuesIterator(rawIter, rawComparator, keyClass, valClass, conf, inputKeyCounter, inputValueCounter); - } @SuppressWarnings("rawtypes") @@ -347,8 +345,9 @@ public Object getCurrentKey() throws IOException { public Iterable getCurrentValues() throws IOException { return valuesIter.getValues(); } - }; + } + ; private static final Set confKeys = new HashSet(); @@ -417,5 +416,4 @@ public Iterable getCurrentValues() throws IOException { public static Set getConfigurationKeySet() { return Collections.unmodifiableSet(confKeys); } - } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedMergedKVInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedMergedKVInput.java index 49d4043b5c..904279b238 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedMergedKVInput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/OrderedGroupedMergedKVInput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -41,7 +41,7 @@ * A {@link MergedLogicalInput} which merges multiple * {@link OrderedGroupedKVInput}s and returns a single view of these by merging * values which belong to the same key. - * + * * Combiners and Secondary Sort are not implemented, so there is no guarantee on * the order of values. */ @@ -84,7 +84,7 @@ private static class OrderedGroupedMergedKeyValuesReader extends KeyValuesReader private Object currentKey; private final MergedInputContext context; - public OrderedGroupedMergedKeyValuesReader(List inputs, MergedInputContext context) + public OrderedGroupedMergedKeyValuesReader(List inputs, MergedInputContext context) throws Exception { keyComparator = ((OrderedGroupedKVInput) inputs.get(0)) .getInputKeyComparator(); @@ -163,7 +163,6 @@ public void discardCurrent() throws IOException { public void moveToNext() throws IOException { iterator.moveToNext(); } - } @SuppressWarnings("unchecked") @@ -224,13 +223,12 @@ public Object next() { public void remove() { throw new UnsupportedOperationException(); } - } /** * Comparator that compares KeyValuesReader on their current key */ - @SuppressWarnings({ "rawtypes", "unchecked" }) + @SuppressWarnings({"rawtypes", "unchecked"}) private static class KVReaderComparator implements Comparator { @@ -251,11 +249,12 @@ public int compare(KeyValuesReader o1, KeyValuesReader o2) { } } } + public float getProgress() throws ProgressFailedException, InterruptedException { float totalProgress = 0.0f; - for(Input input : getInputs()) { - totalProgress += ((OrderedGroupedKVInput)input).getProgress(); + for (Input input : getInputs()) { + totalProgress += ((OrderedGroupedKVInput) input).getProgress(); } - return (1.0f) * totalProgress/getInputs().size(); + return (1.0f) * totalProgress / getInputs().size(); } } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java index 38d5295094..ae46be54df 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/UnorderedKVInput.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.runtime.library.input; @@ -215,7 +215,7 @@ public synchronized List close() throws Exception { if (this.shuffleManager != null) { this.shuffleManager.shutdown(); } - + long dataSize = getContext().getCounters() .findCounter(TaskCounter.SHUFFLE_BYTES_DECOMPRESSED).getValue(); getContext().getStatisticsReporter().reportDataSize(dataSize); @@ -231,10 +231,9 @@ private long getInitialMemoryReq() { getContext().getTotalMemoryAvailableToTask()); } - @SuppressWarnings("rawtypes") private UnorderedKVReader createReader(TezCounter inputRecordCounter, CompressionCodec codec, - int ifileBufferSize, boolean ifileReadAheadEnabled, int ifileReadAheadLength) + int ifileBufferSize, boolean ifileReadAheadEnabled, int ifileReadAheadLength) throws IOException { return new UnorderedKVReader(shuffleManager, conf, codec, ifileReadAheadEnabled, ifileReadAheadLength, ifileBufferSize, inputRecordCounter, getContext()); diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/package-info.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/package-info.java index 9d5d29bdfe..7bc487b1ec 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/package-info.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/input/package-info.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,4 +21,4 @@ package org.apache.tez.runtime.library.input; import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Evolving; \ No newline at end of file +import org.apache.hadoop.classification.InterfaceStability.Evolving; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java index 44cb9d6aae..993c52d66b 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -143,7 +143,7 @@ public synchronized void start() throws Exception { Preconditions.checkArgument(sorterImpl.equals(SorterImpl.PIPELINED), TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED - + "only works with PipelinedSorter."); + + "only works with PipelinedSorter."); } if (sorterImpl.equals(SorterImpl.PIPELINED)) { @@ -195,7 +195,7 @@ public synchronized List close() throws IOException { sorter = null; } else { LOG.warn(getContext().getInputOutputVertexNames() + - ": Attempting to close output {} of type {} before it was started. Generating empty events", + ": Attempting to close output {} of type {} before it was started. Generating empty events", getContext().getDestinationVertexName(), this.getClass().getSimpleName()); returnEvents = generateEmptyEvents(); } @@ -223,7 +223,6 @@ private List generateEmptyEvents() throws IOException { return eventList; } - private static final Set confKeys = new HashSet(); static { diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java index bcacc5238e..f68aeb4485 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.runtime.library.output; @@ -65,7 +65,7 @@ public class UnorderedKVOutput extends AbstractLogicalOutput { @VisibleForTesting Configuration conf; - + private MemoryUpdateCallbackHandler memoryUpdateCallbackHandler; private final AtomicBoolean isStarted = new AtomicBoolean(false); @@ -73,7 +73,6 @@ public UnorderedKVOutput(OutputContext outputContext, int numPhysicalOutputs) { super(outputContext, numPhysicalOutputs); } - @Override public synchronized List initialize() throws Exception { @@ -94,7 +93,7 @@ public synchronized List initialize() UnorderedPartitionedKVWriter.getInitialMemoryRequirement(conf, getContext() .getTotalMemoryAvailableToTask()) : 0; getContext().requestInitialMemory(memRequestSize, memoryUpdateCallbackHandler); - + return Collections.emptyList(); } @@ -131,7 +130,7 @@ public synchronized List close() throws Exception { kvWriter = null; } else { LOG.warn(getContext().getInputOutputVertexNames() + - ": Attempting to close output {} of type {} before it was started. Generating empty events", + ": Attempting to close output {} of type {} before it was started. Generating empty events", getContext().getDestinationVertexName(), this.getClass().getSimpleName()); returnEvents = new LinkedList(); ShuffleUtils diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedPartitionedKVOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedPartitionedKVOutput.java index 9bc7ea40cd..47c54810f5 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedPartitionedKVOutput.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedPartitionedKVOutput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -109,7 +109,7 @@ public synchronized List close() throws Exception { kvWriter = null; } else { LOG.warn(getContext().getInputOutputVertexNames() + - ": Attempting to close output {} of type {} before it was started. Generating empty events", + ": Attempting to close output {} of type {} before it was started. Generating empty events", getContext().getDestinationVertexName(), this.getClass().getSimpleName()); returnEvents = new LinkedList(); ShuffleUtils diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/package-info.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/package-info.java index 4b71be26ec..325f32b8e2 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/package-info.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/package-info.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,4 +21,4 @@ package org.apache.tez.runtime.library.output; import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Evolving; \ No newline at end of file +import org.apache.hadoop.classification.InterfaceStability.Evolving; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/partitioner/HashPartitioner.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/partitioner/HashPartitioner.java index 7a26d38f51..3cdaefa9b5 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/partitioner/HashPartitioner.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/partitioner/HashPartitioner.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -33,5 +33,4 @@ public class HashPartitioner implements Partitioner { public int getPartition(Object key, Object value, int numPartitions) { return (key.hashCode() & Integer.MAX_VALUE) % numPartitions; } - } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/partitioner/RoundRobinPartitioner.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/partitioner/RoundRobinPartitioner.java index 3c3e41aad4..2225bfd76c 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/partitioner/RoundRobinPartitioner.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/partitioner/RoundRobinPartitioner.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,11 +20,11 @@ import org.apache.tez.runtime.library.api.Partitioner; public class RoundRobinPartitioner implements Partitioner { - private int x = 0; + private int x = 0; - @Override - public int getPartition(Object key, Object value, int numPartitions) { - x = x % numPartitions; - return (x++) % numPartitions; - } + @Override + public int getPartition(Object key, Object value, int numPartitions) { + x = x % numPartitions; + return (x++) % numPartitions; + } } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/PreWarmProcessor.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/PreWarmProcessor.java index 4103078dd1..372f0395f3 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/PreWarmProcessor.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/PreWarmProcessor.java @@ -28,7 +28,6 @@ * If this is customized by the user then they need to make sure that * the custom class jar is localized for the prewarm vertex and other * vertices that need to take advantage of prewarming - * */ @Unstable @Public @@ -43,14 +42,14 @@ public void run() throws Exception { preWarmTezCode(); preWarmUserCode(); } - + /** * Pre-warm Tez code. Users can override this with an empty method * to not pre-warm Tez code if they want to. */ protected void preWarmTezCode() { // Do nothing. Can potentially pre-warm Tez library components - + // Currently, must sleep for some time so that container re-use // can be prevented from kicking in. This will allow sufficient // time to obtain containers from YARN as long as those resources @@ -61,13 +60,12 @@ protected void preWarmTezCode() { e.printStackTrace(); } } - + /** - * Pre-warm user code. Users can override this - * to pre-warm their own code if they want to. + * Pre-warm user code. Users can override this + * to pre-warm their own code if they want to. */ protected void preWarmUserCode() { // Do nothing } - } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/SimpleProcessor.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/SimpleProcessor.java index c237bc15e4..09253c190d 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/SimpleProcessor.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/SimpleProcessor.java @@ -35,7 +35,7 @@ /** * Implements an {@link AbstractLogicalIOProcessor} and provides empty * implementations of most methods and handles input/output initialization. - * This can be used to implement simple {@link Processor}s that dont need to + * This can be used to implement simple {@link Processor}s that dont need to * do event handling etc. */ @Public @@ -56,7 +56,7 @@ public void run(Map _inputs, Map _o throws Exception { this.inputs = _inputs; this.outputs = _outputs; - progressHelper = new ProgressHelper(this.inputs, getContext(),this.getClass().getSimpleName()); + progressHelper = new ProgressHelper(this.inputs, getContext(), this.getClass().getSimpleName()); preOp(); run(); postOp(); @@ -65,6 +65,7 @@ public void run(Map _inputs, Map _o /** * Users must implement this method to provide the main * application logic code + * * @throws Exception */ public abstract void run() throws Exception; @@ -72,7 +73,8 @@ public void run(Map _inputs, Map _o /** * Implements input/output initialization. Can be overriden * to implement custom behavior. Called before {@link #run()} - * is called. + * is called. + * * @throws Exception */ protected void preOp() throws Exception { @@ -90,12 +92,13 @@ protected void preOp() throws Exception { } /** - * Called after {@link #run()} is called and can be used to + * Called after {@link #run()} is called and can be used to * do post-processing like committing output etc + * * @throws Exception */ protected void postOp() throws Exception { - //No-op + //No-op } @Override @@ -110,7 +113,7 @@ public void handleEvents(List processorEvents) { @Override public void close() throws Exception { - if( progressHelper != null) { + if (progressHelper != null) { progressHelper.shutDownProgressTaskService(); } } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/SleepProcessor.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/SleepProcessor.java index 3efcd21c92..0315b6358f 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/SleepProcessor.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/SleepProcessor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -58,14 +58,14 @@ public SleepProcessor(ProcessorContext context) { @Override public void initialize() - throws Exception { + throws Exception { if (getContext().getUserPayload() == null) { LOG.info("No processor user payload specified" - + ", using default timeToSleep of 1 ms"); + + ", using default timeToSleep of 1 ms"); timeToSleepMS = 1; } else { SleepProcessorConfig cfg = - new SleepProcessorConfig(); + new SleepProcessorConfig(); cfg.fromUserPayload(getContext().getUserPayload()); timeToSleepMS = cfg.getTimeToSleepMS(); } @@ -77,9 +77,9 @@ public void run(Map _inputs, Map _outputs) throws Exception { inputs = _inputs; outputs = _outputs; - progressHelper = new ProgressHelper(this.inputs, getContext(),this.getClass().getSimpleName()); + progressHelper = new ProgressHelper(this.inputs, getContext(), this.getClass().getSimpleName()); LOG.info("Running the Sleep Processor, sleeping for " - + timeToSleepMS + " ms"); + + timeToSleepMS + " ms"); for (LogicalInput input : _inputs.values()) { input.start(); } @@ -120,7 +120,7 @@ public SleepProcessorConfig() { /** * @param timeToSleepMS Time to sleep in milliseconds */ - public SleepProcessorConfig (int timeToSleepMS) { + public SleepProcessorConfig(int timeToSleepMS) { this.timeToSleepMS = timeToSleepMS; } @@ -137,5 +137,4 @@ public int getTimeToSleepMS() { return timeToSleepMS; } } - } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/resources/WeightedScalingMemoryDistributor.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/resources/WeightedScalingMemoryDistributor.java index 52f3d44b4f..553961230e 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/resources/WeightedScalingMemoryDistributor.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/resources/WeightedScalingMemoryDistributor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -77,7 +77,9 @@ public WeightedScalingMemoryDistributor() { public enum RequestType { PARTITIONED_UNSORTED_OUTPUT, UNSORTED_INPUT, UNSORTED_OUTPUT, SORTED_OUTPUT, SORTED_MERGED_INPUT, PROCESSOR, OTHER - }; + } + + ; private EnumMap typeScaleMap = Maps.newEnumMap(RequestType.class); @@ -89,7 +91,7 @@ public enum RequestType { @Override public Iterable assignMemory(long availableForAllocation, int numTotalInputs, - int numTotalOutputs, Iterable initialRequests) { + int numTotalOutputs, Iterable initialRequests) { // Read in configuration populateTypeScaleMap(); @@ -180,22 +182,22 @@ public Iterable assignMemory(long availableForAllocation, int numTotalInpu } private void adjustAllocationsForNonConcurrent(List allocations, - List requests, int numInputsScaled, long totalInputAllocated, - int numOutputsScaled, long totalOutputAllocated) { + List requests, int numInputsScaled, long totalInputAllocated, + int numOutputsScaled, long totalOutputAllocated) { boolean inputsEnabled = conf.getBoolean( TezConfiguration.TEZ_TASK_SCALE_MEMORY_NON_CONCURRENT_INPUTS_ENABLED, TezConfiguration.TEZ_TASK_SCALE_MEMORY_NON_CONCURRENT_INPUTS_ENABLED_DEFAULT); LOG.info("Adjusting scaled allocations for I/O non-concurrent." - + " numInputsScaled: {} InputAllocated: {} numOutputsScaled: {} outputAllocated: {} inputsEnabled: {}", + + " numInputsScaled: {} InputAllocated: {} numOutputsScaled: {} outputAllocated: {} inputsEnabled: {}", numInputsScaled, totalInputAllocated, numOutputsScaled, totalOutputAllocated, inputsEnabled); for (int i = 0; i < requests.size(); i++) { Request request = requests.get(i); long additional = 0; if (request.componentType == ComponentType.INPUT && inputsEnabled) { - double share = request.requestWeight / (double)numInputsScaled; + double share = request.requestWeight / (double) numInputsScaled; additional = (long) (totalOutputAllocated * share); } else if (request.componentType == ComponentType.OUTPUT) { - double share = request.requestWeight / (double)numOutputsScaled; + double share = request.requestWeight / (double) numOutputsScaled; additional = (long) (totalInputAllocated * share); } if (additional > 0) { @@ -319,7 +321,7 @@ private double computeReservedFraction(int numTotalRequests) { } public static String[] generateWeightStrings(int unsortedPartitioned, int unsorted, - int broadcastIn, int sortedOut, int scatterGatherShuffleIn, int proc, int other) { + int broadcastIn, int sortedOut, int scatterGatherShuffleIn, int proc, int other) { String[] weights = new String[RequestType.values().length]; weights[0] = RequestType.PARTITIONED_UNSORTED_OUTPUT.name() + ":" + unsortedPartitioned; weights[1] = RequestType.UNSORTED_OUTPUT.name() + ":" + unsorted; @@ -343,7 +345,7 @@ public Configuration getConf() { private static class Request { Request(String componentClassname, ComponentType componentType, long requestSize, - RequestType requestType, int requestWeight) { + RequestType requestType, int requestWeight) { this.componentClassname = componentClassname; this.componentType = componentType; this.requestSize = requestSize; diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/BufferUtils.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/BufferUtils.java index a1685edcfa..a3cc88294e 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/BufferUtils.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/BufferUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -75,5 +75,4 @@ public static void copy(DataOutputBuffer src, DataOutputBuffer dst) throws IOExc dst.reset(); dst.write(b1, s1, l1); } - } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/CodecUtils.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/CodecUtils.java index 340ecceea8..d45da6f13a 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/CodecUtils.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/CodecUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -80,7 +80,7 @@ public static CompressionCodec getCodec(Configuration conf) throws IOException { } public static InputStream getDecompressedInputStreamWithBufferSize(CompressionCodec codec, - IFileInputStream checksumIn, Decompressor decompressor, int compressedLength) + IFileInputStream checksumIn, Decompressor decompressor, int compressedLength) throws IOException { String bufferSizeProp = getBufferSizeProperty(codec); CompressionInputStream in = null; @@ -146,14 +146,14 @@ public static Decompressor getDecompressor(CompressionCodec codec) { } public static CompressionInputStream createInputStream(CompressionCodec codec, - InputStream checksumIn, Decompressor decompressor) throws IOException { + InputStream checksumIn, Decompressor decompressor) throws IOException { synchronized (((Configurable) codec).getConf()) { return codec.createInputStream(checksumIn, decompressor); } } public static CompressionOutputStream createOutputStream(CompressionCodec codec, - OutputStream checksumOut, Compressor compressor) throws IOException { + OutputStream checksumOut, Compressor compressor) throws IOException { synchronized (((Configurable) codec).getConf()) { return codec.createOutputStream(checksumOut, compressor); } @@ -165,21 +165,21 @@ public static String getBufferSizeProperty(CompressionCodec codec) { public static String getBufferSizeProperty(String codecClassName) { switch (codecClassName) { - case "org.apache.hadoop.io.compress.DefaultCodec": - case "org.apache.hadoop.io.compress.BZip2Codec": - case "org.apache.hadoop.io.compress.GzipCodec": - return CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY; - case "org.apache.hadoop.io.compress.SnappyCodec": - return CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY; - case "org.apache.hadoop.io.compress.ZStandardCodec": - return CommonConfigurationKeys.IO_COMPRESSION_CODEC_ZSTD_BUFFER_SIZE_KEY; - case "org.apache.hadoop.io.compress.LzoCodec": - case "com.hadoop.compression.lzo.LzoCodec": - return CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZO_BUFFERSIZE_KEY; - case "org.apache.hadoop.io.compress.Lz4Codec": - return CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY; - default: - return null; + case "org.apache.hadoop.io.compress.DefaultCodec": + case "org.apache.hadoop.io.compress.BZip2Codec": + case "org.apache.hadoop.io.compress.GzipCodec": + return CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY; + case "org.apache.hadoop.io.compress.SnappyCodec": + return CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_KEY; + case "org.apache.hadoop.io.compress.ZStandardCodec": + return CommonConfigurationKeys.IO_COMPRESSION_CODEC_ZSTD_BUFFER_SIZE_KEY; + case "org.apache.hadoop.io.compress.LzoCodec": + case "com.hadoop.compression.lzo.LzoCodec": + return CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZO_BUFFERSIZE_KEY; + case "org.apache.hadoop.io.compress.Lz4Codec": + return CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_KEY; + default: + return null; } } @@ -189,21 +189,21 @@ public static int getDefaultBufferSize(Configuration conf, CompressionCodec code public static int getDefaultBufferSize(Configuration conf, String codecClassName) { switch (codecClassName) { - case "org.apache.hadoop.io.compress.DefaultCodec": - case "org.apache.hadoop.io.compress.BZip2Codec": - case "org.apache.hadoop.io.compress.GzipCodec": - return CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT; - case "org.apache.hadoop.io.compress.SnappyCodec": - return CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT; - case "org.apache.hadoop.io.compress.ZStandardCodec": - return CommonConfigurationKeys.IO_COMPRESSION_CODEC_ZSTD_BUFFER_SIZE_DEFAULT; - case "org.apache.hadoop.io.compress.LzoCodec": - case "com.hadoop.compression.lzo.LzoCodec": - return CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZO_BUFFERSIZE_DEFAULT; - case "org.apache.hadoop.io.compress.Lz4Codec": - return CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT; - default: - return DEFAULT_BUFFER_SIZE; + case "org.apache.hadoop.io.compress.DefaultCodec": + case "org.apache.hadoop.io.compress.BZip2Codec": + case "org.apache.hadoop.io.compress.GzipCodec": + return CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT; + case "org.apache.hadoop.io.compress.SnappyCodec": + return CommonConfigurationKeys.IO_COMPRESSION_CODEC_SNAPPY_BUFFERSIZE_DEFAULT; + case "org.apache.hadoop.io.compress.ZStandardCodec": + return CommonConfigurationKeys.IO_COMPRESSION_CODEC_ZSTD_BUFFER_SIZE_DEFAULT; + case "org.apache.hadoop.io.compress.LzoCodec": + case "com.hadoop.compression.lzo.LzoCodec": + return CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZO_BUFFERSIZE_DEFAULT; + case "org.apache.hadoop.io.compress.Lz4Codec": + return CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_BUFFERSIZE_DEFAULT; + default: + return DEFAULT_BUFFER_SIZE; } } -} \ No newline at end of file +} diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/DATA_RANGE_IN_MB.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/DATA_RANGE_IN_MB.java index 75698188e2..924048c55c 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/DATA_RANGE_IN_MB.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/DATA_RANGE_IN_MB.java @@ -18,7 +18,6 @@ package org.apache.tez.runtime.library.utils; - public enum DATA_RANGE_IN_MB { THOUSAND(1000), HUNDRED(100), TEN(10), ONE(1), ZERO(0); diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/FastByteComparisons.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/FastByteComparisons.java index 4bd2552688..6a7099416c 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/FastByteComparisons.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/FastByteComparisons.java @@ -8,9 +8,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -41,22 +41,20 @@ final class FastByteComparisons { * Lexicographically compare two byte arrays. */ public static int compareTo(byte[] b1, int s1, int l1, byte[] b2, int s2, - int l2) { + int l2) { return LexicographicalComparerHolder.BEST_COMPARER.compareTo( b1, s1, l1, b2, s2, l2); } - private interface Comparer { abstract public int compareTo(T buffer1, int offset1, int length1, - T buffer2, int offset2, int length2); + T buffer2, int offset2, int length2); } private static Comparer lexicographicalComparerJavaImpl() { return LexicographicalComparerHolder.PureJavaComparer.INSTANCE; } - /** * Provides a lexicographical comparer implementation; either a Java * implementation or a faster implementation based on {@link sun.misc.Unsafe}. @@ -69,6 +67,7 @@ private static class LexicographicalComparerHolder { LexicographicalComparerHolder.class.getName() + "$UnsafeComparer"; static final Comparer BEST_COMPARER = getBestComparer(); + /** * Returns the Unsafe-using Comparer, or falls back to the pure-Java * implementation if unable to do so. @@ -92,7 +91,7 @@ private enum PureJavaComparer implements Comparer { @Override public int compareTo(byte[] buffer1, int offset1, int length1, - byte[] buffer2, int offset2, int length2) { + byte[] buffer2, int offset2, int length2) { // Short circuit equal case if (buffer1 == buffer2 && offset1 == offset2 && @@ -173,7 +172,7 @@ static boolean lessThanUnsigned(long x1, long x2) { */ @Override public int compareTo(byte[] buffer1, int offset1, int length1, - byte[] buffer2, int offset2, int length2) { + byte[] buffer2, int offset2, int length2) { // Short circuit equal case if (buffer1 == buffer2 && offset1 == offset2 && diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/Grouper.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/Grouper.java index 84900f8f01..fb0e8ecd90 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/Grouper.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/Grouper.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -37,11 +37,11 @@ public class Grouper { public Grouper init(int numItem, int numGroup) { Preconditions.checkArgument(numGroup > 0, - "Number of groups is " + numGroup + ". Should be positive"); + "Number of groups is " + numGroup + ". Should be positive"); Preconditions.checkArgument(numItem > 0, - "Number of items is " + numItem + ". Should be positive"); + "Number of items is " + numItem + ". Should be positive"); Preconditions.checkArgument(numItem >= numGroup, - "Num of groups + " + numGroup + " shouldn't be more than number of items " + numItem); + "Num of groups + " + numGroup + " shouldn't be more than number of items " + numItem); this.numItem = numItem; this.numGroup = numGroup; this.itemPerGroup1 = numItem / numGroup; @@ -74,7 +74,7 @@ public int getLastItemInGroup(int groupId) { public int getGroupId(int itemId) { Preconditions.checkArgument(0 <= itemId && itemId < numItem, "Invalid itemId" + itemId); if (itemId < itemPerGroup1 * numGroup1) { - return itemId/ itemPerGroup1; + return itemId / itemPerGroup1; } else { return numGroup1 + (itemId - itemPerGroup1 * numGroup1) / itemPerGroup2; } @@ -84,6 +84,6 @@ public boolean isInGroup(int itemId, int groupId) { Preconditions.checkArgument(0 <= groupId && groupId < numGroup, "Invalid groupId" + groupId); Preconditions.checkArgument(0 <= itemId && itemId < numItem, "Invalid itemId" + itemId); return getFirstItemInGroup(groupId) <= itemId - && itemId < getFirstItemInGroup(groupId) + getNumItemsInGroup(groupId); + && itemId < getFirstItemInGroup(groupId) + getNumItemsInGroup(groupId); } } diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/LocalProgress.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/LocalProgress.java index 8aa1f62e7f..08a13e144b 100644 --- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/LocalProgress.java +++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/utils/LocalProgress.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestFairShuffleVertexManager.java b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestFairShuffleVertexManager.java index 5108b8f9e6..df25cc6996 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestFairShuffleVertexManager.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestFairShuffleVertexManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -51,7 +51,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -@SuppressWarnings({ "unchecked", "rawtypes" }) +@SuppressWarnings({"unchecked", "rawtypes"}) public class TestFairShuffleVertexManager extends TestShuffleVertexManagerUtils { List emptyCompletions = null; @@ -64,7 +64,7 @@ public void testAutoParallelismConfig() throws Exception { final VertexManagerPluginContext mockContext = createVertexManagerContext( "Vertex1", 2, "Vertex2", 2, "Vertex3", 2, - "Vertex4", 4, scheduledTasks, null); + "Vertex4", 4, scheduledTasks, null); manager = createManager(null, mockContext, null, 0.5f); verify(mockContext, times(1)).vertexReconfigurationPlanned(); // Tez notified of reconfig @@ -117,23 +117,23 @@ public void testReduceSchedulingWithPartitionStats() throws Exception { long[] partitionStats = new long[]{(MB), (2 * MB), (5 * MB)}; testSchedulingWithPartitionStats( FairRoutingType.REDUCE_PARALLELISM, numScatherAndGatherSourceTasks, - partitionStats, 2,2, 2, newEdgeManagers); + partitionStats, 2, 2, 2, newEdgeManagers); EdgeManagerPluginOnDemand edgeManager = - (EdgeManagerPluginOnDemand)newEdgeManagers.values().iterator().next(); + (EdgeManagerPluginOnDemand) newEdgeManagers.values().iterator().next(); // The first destination task fetches two partitions from all source tasks. // Thus the # of inputs == # of source tasks * 2 merged partitions Assert.assertEquals(numScatherAndGatherSourceTasks * 2, edgeManager.getNumDestinationTaskPhysicalInputs(0)); for (int sourceTaskIndex = 0; - sourceTaskIndex < numScatherAndGatherSourceTasks; sourceTaskIndex++) { + sourceTaskIndex < numScatherAndGatherSourceTasks; sourceTaskIndex++) { for (int j = 0; j < 2; j++) { if (j == 0) { EdgeManagerPluginOnDemand.CompositeEventRouteMetadata routeMetadata = edgeManager.routeCompositeDataMovementEventToDestination(sourceTaskIndex, 0); Assert.assertEquals(2, routeMetadata.getCount()); Assert.assertEquals(0, routeMetadata.getSource()); - Assert.assertEquals(sourceTaskIndex*2, routeMetadata.getTarget()); + Assert.assertEquals(sourceTaskIndex * 2, routeMetadata.getTarget()); } else { EdgeManagerPluginOnDemand.EventRouteMetadata routeMetadata = edgeManager.routeInputSourceTaskFailedEventToDestination(sourceTaskIndex, 0); @@ -160,21 +160,21 @@ public void testFairSchedulingWithPartitionStats() throws Exception { // Get the first edgeManager which is SCATTER_GATHER. EdgeManagerPluginOnDemand edgeManager = - (EdgeManagerPluginOnDemand)newEdgeManagers.values().iterator().next(); + (EdgeManagerPluginOnDemand) newEdgeManagers.values().iterator().next(); // The first destination task fetches two partitions from all source tasks. // Thus the # of inputs == # of source tasks * 2 merged partitions Assert.assertEquals(numScatherAndGatherSourceTasks * 2, edgeManager.getNumDestinationTaskPhysicalInputs(0)); for (int sourceTaskIndex = 0; sourceTaskIndex < numScatherAndGatherSourceTasks; - sourceTaskIndex++) { + sourceTaskIndex++) { for (int j = 0; j < 2; j++) { if (j == 0) { EdgeManagerPluginOnDemand.CompositeEventRouteMetadata routeMetadata = edgeManager.routeCompositeDataMovementEventToDestination(sourceTaskIndex, 0); Assert.assertEquals(2, routeMetadata.getCount()); Assert.assertEquals(0, routeMetadata.getSource()); - Assert.assertEquals(sourceTaskIndex*2, routeMetadata.getTarget()); + Assert.assertEquals(sourceTaskIndex * 2, routeMetadata.getTarget()); } else { EdgeManagerPluginOnDemand.EventRouteMetadata routeMetadata = edgeManager.routeInputSourceTaskFailedEventToDestination(sourceTaskIndex, 0); @@ -210,7 +210,7 @@ public void testFairSchedulingWithPartitionStats() throws Exception { Assert.assertEquals(numScatherAndGatherSourceTasks / 2, edgeManager.getNumDestinationTaskPhysicalInputs(2)); for (int sourceTaskIndex = numScatherAndGatherSourceTasks / 2; - sourceTaskIndex < numScatherAndGatherSourceTasks; sourceTaskIndex++) { + sourceTaskIndex < numScatherAndGatherSourceTasks; sourceTaskIndex++) { for (int j = 0; j < 2; j++) { if (j == 0) { EdgeManagerPluginOnDemand.CompositeEventRouteMetadata routeMetadata = @@ -235,7 +235,7 @@ public void testFairSchedulingWithPartitionStats() throws Exception { public void testOverflow() throws Exception { final int numScatherAndGatherSourceTasks = 30000; final Map newEdgeManagers = - new HashMap(); + new HashMap(); final int firstPartitionSize = 1; final int secondPartitionSize = 2; final int thirdPartitionSize = 500; @@ -243,7 +243,7 @@ public void testOverflow() throws Exception { (secondPartitionSize * MB), (thirdPartitionSize * MB)}; final int expectedDestinationTasks = (firstPartitionSize + secondPartitionSize + thirdPartitionSize) - * numScatherAndGatherSourceTasks / 1000; + * numScatherAndGatherSourceTasks / 1000; testSchedulingWithPartitionStats( FairRoutingType.FAIR_PARALLELISM, @@ -332,7 +332,6 @@ private void testSchedulingWithPartitionStats( Assert.assertTrue(manager.pendingTasks.size() == numOfTasksInDestination); // no tasks scheduled Assert.assertTrue(manager.totalNumBipartiteSourceTasks == numOfTasksInr1); - for (int i = 0; i < numCompletedEvents; i++) { VertexManagerEvent vmEvent = getVertexManagerEvent(partitionStats, 0, r1, true); @@ -353,7 +352,7 @@ private void testSchedulingWithPartitionStats( Assert.assertEquals(1, newEdgeManagers.size()); EdgeManagerPluginOnDemand edgeManager = - (EdgeManagerPluginOnDemand)newEdgeManagers.values().iterator().next(); + (EdgeManagerPluginOnDemand) newEdgeManagers.values().iterator().next(); // For each source task, there are 3 outputs, // the same as original number of partitions. for (int i = 0; i < numOfTasksInr1; i++) { @@ -362,23 +361,23 @@ private void testSchedulingWithPartitionStats( } for (int sourceTaskIndex = 0; sourceTaskIndex < numOfTasksInr1; - sourceTaskIndex++) { + sourceTaskIndex++) { Assert.assertEquals(expectedNumDestinationConsumerTasks, edgeManager.getNumDestinationConsumerTasks(sourceTaskIndex)); } } private static FairShuffleVertexManager createManager(Configuration conf, - VertexManagerPluginContext context, Float min, Float max) { + VertexManagerPluginContext context, Float min, Float max) { return createManager(conf, context, true, 1000l * MB, min, max); } private static FairShuffleVertexManager createManager(Configuration conf, - VertexManagerPluginContext context, - Boolean enableAutoParallelism, Long desiredTaskInputSize, Float min, - Float max) { - return (FairShuffleVertexManager)TestShuffleVertexManagerBase.createManager( + VertexManagerPluginContext context, + Boolean enableAutoParallelism, Long desiredTaskInputSize, Float min, + Float max) { + return (FairShuffleVertexManager) TestShuffleVertexManagerBase.createManager( FairShuffleVertexManager.class, conf, context, enableAutoParallelism, - desiredTaskInputSize, min, max); + desiredTaskInputSize, min, max); } } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestInputReadyVertexManager.java b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestInputReadyVertexManager.java index 2eaaba4871..1f7ea9f651 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestInputReadyVertexManager.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestInputReadyVertexManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -47,18 +47,18 @@ @SuppressWarnings("unchecked") public class TestInputReadyVertexManager { - + @Captor ArgumentCaptor> requestCaptor; - + @Before public void init() { MockitoAnnotations.initMocks(this); } - @Test (timeout=5000) + @Test(timeout = 5000) public void testBasicScatterGather() throws Exception { - HashMap mockInputVertices = + HashMap mockInputVertices = new HashMap(); String mockSrcVertexId1 = "Vertex1"; EdgeProperty eProp1 = EdgeProperty.create( @@ -67,9 +67,9 @@ public void testBasicScatterGather() throws Exception { SchedulingType.SEQUENTIAL, OutputDescriptor.create("out"), InputDescriptor.create("in")); - + String mockManagedVertexId = "Vertex"; - + VertexManagerPluginContext mockContext = mock(VertexManagerPluginContext.class); when(mockContext.getInputVertexEdgeProperties()).thenReturn(mockInputVertices); when(mockContext.getVertexName()).thenReturn(mockManagedVertexId); @@ -95,10 +95,10 @@ public void testBasicScatterGather() throws Exception { verify(mockContext, times(1)).scheduleTasks(requestCaptor.capture()); Assert.assertEquals(2, requestCaptor.getValue().size()); } - - @Test (timeout=5000) + + @Test(timeout = 5000) public void testBasicOneToOne() throws Exception { - HashMap mockInputVertices = + HashMap mockInputVertices = new HashMap(); String mockSrcVertexId1 = "Vertex1"; EdgeProperty eProp1 = EdgeProperty.create( @@ -107,16 +107,16 @@ public void testBasicOneToOne() throws Exception { SchedulingType.SEQUENTIAL, OutputDescriptor.create("out"), InputDescriptor.create("in")); - + String mockManagedVertexId = "Vertex"; - + VertexManagerPluginContext mockContext = mock(VertexManagerPluginContext.class); when(mockContext.getInputVertexEdgeProperties()).thenReturn(mockInputVertices); when(mockContext.getVertexName()).thenReturn(mockManagedVertexId); when(mockContext.getVertexNumTasks(mockManagedVertexId)).thenReturn(3); when(mockContext.getVertexNumTasks(mockSrcVertexId1)).thenReturn(3); mockInputVertices.put(mockSrcVertexId1, eProp1); - + InputReadyVertexManager manager = new InputReadyVertexManager(mockContext); manager.initialize(); verify(mockContext, times(1)).vertexReconfigurationPlanned(); @@ -152,10 +152,10 @@ public void testBasicOneToOne() throws Exception { Assert.assertEquals(2, requestCaptor.getValue().get(0) .getTaskLocationHint().getAffinitizedTask().getTaskIndex()); } - - @Test (timeout=5000) + + @Test(timeout = 5000) public void testDelayedConfigureOneToOne() throws Exception { - HashMap mockInputVertices = + HashMap mockInputVertices = new HashMap(); String mockSrcVertexId1 = "Vertex1"; EdgeProperty eProp1 = EdgeProperty.create( @@ -164,16 +164,16 @@ public void testDelayedConfigureOneToOne() throws Exception { SchedulingType.SEQUENTIAL, OutputDescriptor.create("out"), InputDescriptor.create("in")); - + String mockManagedVertexId = "Vertex"; - + VertexManagerPluginContext mockContext = mock(VertexManagerPluginContext.class); when(mockContext.getInputVertexEdgeProperties()).thenReturn(mockInputVertices); when(mockContext.getVertexName()).thenReturn(mockManagedVertexId); when(mockContext.getVertexNumTasks(mockManagedVertexId)).thenReturn(3); when(mockContext.getVertexNumTasks(mockSrcVertexId1)).thenReturn(3); mockInputVertices.put(mockSrcVertexId1, eProp1); - + InputReadyVertexManager manager = new InputReadyVertexManager(mockContext); manager.initialize(); verify(mockContext, times(1)).vertexReconfigurationPlanned(); @@ -189,7 +189,7 @@ public void testDelayedConfigureOneToOne() throws Exception { // then source vertex configured. now we start manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED)); verify(mockContext, times(1)).doneReconfiguringVertex(); - + verify(mockContext, times(2)).scheduleTasks(requestCaptor.capture()); manager.onSourceTaskCompleted( TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId1, 2)); @@ -202,9 +202,9 @@ public void testDelayedConfigureOneToOne() throws Exception { .getTaskLocationHint().getAffinitizedTask().getTaskIndex()); } - @Test (timeout=5000) + @Test(timeout = 5000) public void testComplex() throws Exception { - HashMap mockInputVertices = + HashMap mockInputVertices = new HashMap(); String mockSrcVertexId1 = "Vertex1"; EdgeProperty eProp1 = EdgeProperty.create( @@ -227,7 +227,7 @@ public void testComplex() throws Exception { SchedulingType.SEQUENTIAL, OutputDescriptor.create("out"), InputDescriptor.create("in")); - + String mockManagedVertexId = "Vertex"; Container mockContainer2 = mock(Container.class); ContainerId mockCId2 = mock(ContainerId.class); @@ -235,7 +235,7 @@ public void testComplex() throws Exception { Container mockContainer3 = mock(Container.class); ContainerId mockCId3 = mock(ContainerId.class); when(mockContainer3.getId()).thenReturn(mockCId3); - + VertexManagerPluginContext mockContext = mock(VertexManagerPluginContext.class); when(mockContext.getInputVertexEdgeProperties()).thenReturn(mockInputVertices); when(mockContext.getVertexName()).thenReturn(mockManagedVertexId); @@ -245,9 +245,9 @@ public void testComplex() throws Exception { mockInputVertices.put(mockSrcVertexId1, eProp1); mockInputVertices.put(mockSrcVertexId2, eProp2); mockInputVertices.put(mockSrcVertexId3, eProp3); - + List initialCompletions = Lists.newArrayList(); - + // 1-1 sources do not match managed tasks. setParallelism called to make them match when(mockContext.getVertexNumTasks(mockManagedVertexId)).thenReturn(4); InputReadyVertexManager manager = new InputReadyVertexManager(mockContext); @@ -259,7 +259,7 @@ public void testComplex() throws Exception { verify(mockContext, times(1)).reconfigureVertex(3, null, null); verify(mockContext, times(1)).doneReconfiguringVertex(); manager.onVertexStarted(initialCompletions); - + // 1-1 sources do not match when(mockContext.getVertexNumTasks(mockManagedVertexId)).thenReturn(3); when(mockContext.getVertexNumTasks(mockSrcVertexId3)).thenReturn(4); @@ -275,9 +275,9 @@ public void testComplex() throws Exception { e.getMessage().contains("1-1 source vertices must have identical concurrency"); } verify(mockContext, times(1)).reconfigureVertex(anyInt(), any(), any()); // not invoked - + when(mockContext.getVertexNumTasks(mockSrcVertexId3)).thenReturn(3); - + initialCompletions.add(TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId1, 0)); initialCompletions.add(TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId2, 0)); manager = new InputReadyVertexManager(mockContext); @@ -332,11 +332,10 @@ public void testComplex() throws Exception { .getTaskLocationHint().getAffinitizedTask().getVertexName()); Assert.assertEquals(2, requestCaptor.getValue().get(0) .getTaskLocationHint().getAffinitizedTask().getTaskIndex()); // affinity to last completion - + // no more starts manager.onSourceTaskCompleted( TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId3, 2)); verify(mockContext, times(3)).scheduleTasks(anyList()); - } } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java index fcbcb42ac4..cfa78d2263 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -47,7 +47,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -@SuppressWarnings({ "unchecked", "rawtypes" }) +@SuppressWarnings({"unchecked", "rawtypes"}) public class TestShuffleVertexManager extends TestShuffleVertexManagerUtils { List emptyCompletions = null; @@ -100,7 +100,7 @@ public void testLargeDataSize() throws IOException { // Ensure long overflow doesn't reduce mistakenly // Overflow can occur previously when output size * num tasks for a single vertex would over flow max long // - manager = createManager(conf, mockContext, true, (long)(Long.MAX_VALUE / 1.5), 1.0f, 1.0f); + manager = createManager(conf, mockContext, true, (long) (Long.MAX_VALUE / 1.5), 1.0f, 1.0f); manager.onVertexStarted(emptyCompletions); manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId1, VertexState.CONFIGURED)); manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId2, VertexState.CONFIGURED)); @@ -130,7 +130,7 @@ public void testLargeDataSize() throws IOException { Assert.assertEquals(2, manager.numBipartiteSourceTasksCompleted); Assert.assertEquals(0L, manager.completedSourceTasksOutputSize); // First source 2 task completes - vmEvent = getVertexManagerEvent(null, Long.MAX_VALUE >> 1 , mockSrcVertexId2); + vmEvent = getVertexManagerEvent(null, Long.MAX_VALUE >> 1, mockSrcVertexId2); manager.onVertexManagerEventReceived(vmEvent); manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId2, 0)); Assert.assertEquals(4, manager.pendingTasks.size()); @@ -138,7 +138,7 @@ public void testLargeDataSize() throws IOException { Assert.assertEquals(3, manager.numBipartiteSourceTasksCompleted); Assert.assertEquals(Long.MAX_VALUE >> 1, manager.completedSourceTasksOutputSize); // Second source 2 task completes - vmEvent = getVertexManagerEvent(null, Long.MAX_VALUE >> 1 , mockSrcVertexId2); + vmEvent = getVertexManagerEvent(null, Long.MAX_VALUE >> 1, mockSrcVertexId2); manager.onVertexManagerEventReceived(vmEvent); manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId2, 1)); // Auto-reduce is triggered @@ -180,7 +180,6 @@ public void testAutoParallelismConfig() throws Exception { .setMinTaskParallelism(10).setSlowStartMaxSrcCompletionFraction(0.5f).build(); when(mockContext.getUserPayload()).thenReturn(pluginDesc.getUserPayload()); - manager = ReflectionUtils.createClazzInstance(pluginDesc.getClassName(), new Class[]{VertexManagerPluginContext.class}, new Object[]{mockContext}); manager.initialize(); @@ -301,17 +300,16 @@ public void testSchedulingWithPartitionStats() throws IOException { Assert.assertTrue(scheduledTasks.get(2) == 1); } - private static ShuffleVertexManager createManager(Configuration conf, - VertexManagerPluginContext context, Float min, Float max) { + VertexManagerPluginContext context, Float min, Float max) { return createManager(conf, context, true, 1000l, min, max); } private static ShuffleVertexManager createManager(Configuration conf, - VertexManagerPluginContext context, - Boolean enableAutoParallelism, Long desiredTaskInputSize, Float min, - Float max) { - return (ShuffleVertexManager)TestShuffleVertexManagerBase.createManager( + VertexManagerPluginContext context, + Boolean enableAutoParallelism, Long desiredTaskInputSize, Float min, + Float max) { + return (ShuffleVertexManager) TestShuffleVertexManagerBase.createManager( ShuffleVertexManager.class, conf, context, enableAutoParallelism, desiredTaskInputSize, min, max); } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManagerBase.java b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManagerBase.java index 75866248c3..2aff6cb4eb 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManagerBase.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManagerBase.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -59,7 +59,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -@SuppressWarnings({ "unchecked", "rawtypes" }) +@SuppressWarnings({"unchecked", "rawtypes"}) @RunWith(Parameterized.class) public class TestShuffleVertexManagerBase extends TestShuffleVertexManagerUtils { @@ -279,8 +279,8 @@ public void testTez978() throws IOException { final List scheduledTasks = Lists.newLinkedList(); final VertexManagerPluginContext mockContext = createVertexManagerContext( - mockSrcVertexId1, 2, mockSrcVertexId2, 2, mockSrcVertexId3, 2, - mockManagedVertexId, 4, scheduledTasks, null); + mockSrcVertexId1, 2, mockSrcVertexId2, 2, mockSrcVertexId3, 2, + mockManagedVertexId, 4, scheduledTasks, null); //min/max fraction of 0.01/0.75 would ensure that we hit determineParallelism code path on receiving first event itself. manager = createManager(conf, mockContext, 0.01f, 0.75f); @@ -346,14 +346,14 @@ public void testTez978() throws IOException { Assert.assertEquals(40, manager.totalNumBipartiteSourceTasks); Assert.assertEquals(0, manager.numBipartiteSourceTasksCompleted); //send 8 events with payload size as 10MB - for(int i=0;i<8;i++) { + for (int i = 0; i < 8; i++) { //small payload - create new event each time or it will be ignored (from same task) manager.onVertexManagerEventReceived(getVertexManagerEvent(null, 10 * MB, mockSrcVertexId1)); manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId1, i)); //should not change parallelism verify(mockContext, times(1)).reconfigureVertex(anyInt(), any(), anyMap()); } - for(int i=0;i<3;i++) { + for (int i = 0; i < 3; i++) { manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertexId2, i)); verify(mockContext, times(1)).reconfigureVertex(anyInt(), any(), anyMap()); } @@ -377,11 +377,11 @@ public void testAutoParallelism() throws Exception { final List scheduledTasks = Lists.newLinkedList(); final Map newEdgeManagers = - new HashMap(); + new HashMap(); final VertexManagerPluginContext mockContext = createVertexManagerContext( - mockSrcVertexId1, 2, mockSrcVertexId2, 2, mockSrcVertexId3, 2, - mockManagedVertexId, 4, scheduledTasks, newEdgeManagers); + mockSrcVertexId1, 2, mockSrcVertexId2, 2, mockSrcVertexId3, 2, + mockManagedVertexId, 4, scheduledTasks, newEdgeManagers); // parallelism changed due to small data size manager = createManager(conf, mockContext, 0.5f, 0.5f); @@ -433,7 +433,7 @@ public void testAutoParallelism() throws Exception { Assert.assertEquals(2, newEdgeManagers.size()); EdgeManagerPluginOnDemand edgeManager = - (EdgeManagerPluginOnDemand)newEdgeManagers.values().iterator().next(); + (EdgeManagerPluginOnDemand) newEdgeManagers.values().iterator().next(); // 4 source task outputs - same as original number of partitions Assert.assertEquals(4, edgeManager.getNumSourceTaskPhysicalOutputs(0)); @@ -572,14 +572,14 @@ public void testShuffleVertexManagerSlowStart() { manager.onVertexStateUpdated(new VertexStateUpdate(mockSrcVertexId3, VertexState.CONFIGURED)); Assert.assertEquals(3, manager.pendingTasks.size()); - Assert.assertEquals(numTasks*2, manager.totalNumBipartiteSourceTasks); + Assert.assertEquals(numTasks * 2, manager.totalNumBipartiteSourceTasks); Assert.assertEquals(0, manager.numBipartiteSourceTasksCompleted); float completedTasksThreshold = 0.8f * numTasks; // Finish all tasks before exceeding the threshold - for (String mockSrcVertex : new String[] { mockSrcVertexId1, mockSrcVertexId2 }) { + for (String mockSrcVertex : new String[]{mockSrcVertexId1, mockSrcVertexId2}) { for (int i = 0; i < mockContext.getVertexNumTasks(mockSrcVertex); ++i) { // complete 0th tasks outside the loop - manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertex, i+1)); + manager.onSourceTaskCompleted(createTaskAttemptIdentifier(mockSrcVertex, i + 1)); if ((i + 2) >= completedTasksThreshold) { // stop before completing more than min/max source tasks break; @@ -1012,7 +1012,6 @@ public void test_Tez1649_with_mixed_edges() { Assert.assertTrue(manager.pendingTasks.size() == 0); // all tasks scheduled Assert.assertTrue(scheduledTasks.size() == 3); - //try with a zero task vertex (with non-scatter-gather edges) scheduledTasks.clear(); manager = createManager(conf, mockContext, 0.001f, 0.001f); @@ -1103,8 +1102,7 @@ public void testZeroTasksSendsConfigured() throws IOException { verify(mockContext).doneReconfiguringVertex(); } - - @Test(timeout=5000) + @Test(timeout = 5000) public void testTezDrainCompletionsOnVertexStart() throws IOException { Configuration conf = new Configuration(); ShuffleVertexManagerBase manager; @@ -1117,20 +1115,19 @@ public void testTezDrainCompletionsOnVertexStart() throws IOException { final List scheduledTasks = Lists.newLinkedList(); final VertexManagerPluginContext mockContext = createVertexManagerContext( - mockSrcVertexId1, 2, mockSrcVertexId2, 2, mockSrcVertexId3, 2, - mockManagedVertexId, 4, scheduledTasks, null); + mockSrcVertexId1, 2, mockSrcVertexId2, 2, mockSrcVertexId3, 2, + mockManagedVertexId, 4, scheduledTasks, null); //min/max fraction of 0.01/0.75 would ensure that we hit determineParallelism code path on receiving first event itself. manager = createManager(conf, mockContext, 0.01f, 0.75f); Assert.assertEquals(0, manager.numBipartiteSourceTasksCompleted); manager.onVertexStarted(Collections.singletonList( - TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId1, 0))); + TestShuffleVertexManager.createTaskAttemptIdentifier(mockSrcVertexId1, 0))); Assert.assertEquals(1, manager.numBipartiteSourceTasksCompleted); - } private ShuffleVertexManagerBase createManager(Configuration conf, - VertexManagerPluginContext context, Float min, Float max) { + VertexManagerPluginContext context, Float min, Float max) { return createManager(this.shuffleVertexManagerClass, conf, context, true, null, min, max); } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManagerUtils.java b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManagerUtils.java index 44adc462bc..1c1694b418 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManagerUtils.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManagerUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,7 +18,6 @@ package org.apache.tez.dag.library.vertexmanager; - import com.google.protobuf.ByteString; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.DataOutputBuffer; @@ -63,7 +62,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -@SuppressWarnings({ "unchecked", "rawtypes" }) +@SuppressWarnings({"unchecked", "rawtypes"}) public class TestShuffleVertexManagerUtils { static long MB = 1024l * 1024l; TezVertexID vertexId = TezVertexID.fromString("vertex_1436907267600_195589_1_00"); @@ -117,12 +116,12 @@ VertexManagerPluginContext createVertexManagerContext( } VertexManagerEvent getVertexManagerEvent(long[] sizes, - long inputSize, String vertexName) throws IOException { + long inputSize, String vertexName) throws IOException { return getVertexManagerEvent(sizes, inputSize, vertexName, false); } VertexManagerEvent getVertexManagerEvent(long[] partitionSizes, - long uncompressedTotalSize, String vertexName, boolean reportDetailedStats) + long uncompressedTotalSize, String vertexName, boolean reportDetailedStats) throws IOException { ByteBuffer payload; long totalSize = 0; @@ -139,7 +138,7 @@ VertexManagerEvent getVertexManagerEvent(long[] partitionSizes, partitionStats.serialize(dout); ByteString partitionStatsBytes = TezCommonUtils.compressByteArrayToByteString( - dout.getData()); + dout.getData()); if (reportDetailedStats) { payload = VertexManagerEventPayloadProto.newBuilder() .setOutputSize(totalSize) @@ -155,7 +154,6 @@ VertexManagerEvent getVertexManagerEvent(long[] partitionSizes, .build().toByteString() .asReadOnlyByteBuffer(); } - } else { payload = VertexManagerEventPayloadProto.newBuilder() .setOutputSize(totalSize) @@ -219,7 +217,7 @@ static ShuffleVertexManager createShuffleVertexManager( if (min != null) { conf.setFloat( ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, - min); + min); } else { conf.unset( ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION); @@ -227,7 +225,7 @@ static ShuffleVertexManager createShuffleVertexManager( if (max != null) { conf.setFloat( ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION, - max); + max); } else { conf.unset(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION); } @@ -286,15 +284,17 @@ static FairShuffleVertexManager createFairShuffleVertexManager( protected static class ScheduledTasksAnswer implements Answer { private List scheduledTasks; + public ScheduledTasksAnswer(List scheduledTasks) { this.scheduledTasks = scheduledTasks; } + @Override public Object answer(InvocationOnMock invocation) throws IOException { Object[] args = invocation.getArguments(); scheduledTasks.clear(); List tasks = - (List)args[0]; + (List) args[0]; for (VertexManagerPluginContext.ScheduleTaskRequest task : tasks) { scheduledTasks.add(task.getTaskIndex()); } @@ -307,14 +307,15 @@ protected static class reconfigVertexAnswer implements Answer { private VertexManagerPluginContext mockContext; private String mockManagedVertexId; private Map newEdgeManagers; + public reconfigVertexAnswer(VertexManagerPluginContext mockContext, - String mockManagedVertexId, - Map newEdgeManagers) { + String mockManagedVertexId, + Map newEdgeManagers) { this.mockContext = mockContext; this.mockManagedVertexId = mockManagedVertexId; this.newEdgeManagers = newEdgeManagers; - } + public Object answer(InvocationOnMock invocation) throws Exception { final int numTasks = ((Integer) invocation.getArguments()[0]).intValue(); when(mockContext.getVertexNumTasks(mockManagedVertexId)).thenReturn(numTasks); @@ -359,8 +360,8 @@ public String getVertexGroupName() { }; if (newEdgeManagers != null) { EdgeManagerPlugin edgeManager = ReflectionUtils - .createClazzInstance(pluginDesc.getClassName(), - new Class[]{EdgeManagerPluginContext.class}, new Object[]{emContext}); + .createClazzInstance(pluginDesc.getClassName(), + new Class[]{EdgeManagerPluginContext.class}, new Object[]{emContext}); edgeManager.initialize(); newEdgeManagers.put(entry.getKey(), edgeManager); } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestVertexManagerWithConcurrentInput.java b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestVertexManagerWithConcurrentInput.java index 619a4cdd1d..339ec0d12a 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestVertexManagerWithConcurrentInput.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestVertexManagerWithConcurrentInput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-runtime-library/src/test/java/org/apache/tez/http/TestHttpConnection.java b/tez-runtime-library/src/test/java/org/apache/tez/http/TestHttpConnection.java index afeb6e561f..0560d6c5c1 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/http/TestHttpConnection.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/http/TestHttpConnection.java @@ -124,7 +124,7 @@ public void testAsyncHttpConnectionInterrupt() AsyncHttpConnection asyncHttpConn = getAsyncHttpConnection(params); Future future = executorService.submit(new Worker(latch, asyncHttpConn, true)); - while(currentThread == null) { + while (currentThread == null) { synchronized (this) { wait(100); } @@ -179,7 +179,7 @@ public Void call() throws Exception { currentThread = Thread.currentThread(); connection.connect(); fail(); - } catch(Throwable t) { + } catch (Throwable t) { if (expectingInterrupt) { if (t instanceof ConnectException) { //ClosedByInterruptException via NettyConnectListener.operationComplete() diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/common/resources/TestWeightedScalingMemoryDistributor.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/common/resources/TestWeightedScalingMemoryDistributor.java index 2fbe26443c..2e38a95d79 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/common/resources/TestWeightedScalingMemoryDistributor.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/common/resources/TestWeightedScalingMemoryDistributor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -43,7 +43,7 @@ import com.google.common.base.Joiner; public class TestWeightedScalingMemoryDistributor extends TestMemoryDistributor { - + @Override public void setup() { conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ENABLED, true); @@ -52,7 +52,7 @@ public void setup() { conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_RESERVE_FRACTION, 0.3d); conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ADDITIONAL_RESERVATION_FRACTION_PER_IO, 0.0d); } - + @Test(timeout = 5000) public void testSimpleWeightedScaling() throws TezException { Configuration conf = new Configuration(this.conf); @@ -146,7 +146,7 @@ public void testAdditionalReserveFractionWeightedScaling() throws TezException { assertEquals(500, e3Callback.assigned); assertEquals(1500, e4Callback.assigned); } - + @Test(timeout = 5000) public void testWeightedScalingNonConcurrent() throws TezException { Configuration conf = new Configuration(this.conf); @@ -331,5 +331,4 @@ private OutputDescriptor createTestOutputDescriptor(Class c = TezRuntimeConfiguration.class; @@ -51,5 +49,4 @@ public void testKeySet() throws IllegalAccessException { } assertTrue("Missing keys in key set: " + expectedKeys, expectedKeys.size() == 0); } - } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductCombination.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductCombination.java index 4193aec709..cc4e113c4f 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductCombination.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductCombination.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,39 +34,39 @@ private void verifyCombination(CartesianProductCombination combination, int[] re } private void testCombinationTwoWayVertex0() { - CartesianProductCombination combination = new CartesianProductCombination(new int[]{2,3}, 0); + CartesianProductCombination combination = new CartesianProductCombination(new int[]{2, 3}, 0); combination.firstTaskWithFixedChunk(1); - verifyCombination(combination, new int[]{1,0}, 3); + verifyCombination(combination, new int[]{1, 0}, 3); assertTrue(combination.nextTaskWithFixedChunk()); - verifyCombination(combination, new int[]{1,1}, 4); + verifyCombination(combination, new int[]{1, 1}, 4); assertTrue(combination.nextTaskWithFixedChunk()); - verifyCombination(combination, new int[]{1,2}, 5); + verifyCombination(combination, new int[]{1, 2}, 5); assertFalse(combination.nextTaskWithFixedChunk()); } private void testCombinationTwoWayVertex1() { - CartesianProductCombination combination = new CartesianProductCombination(new int[]{2,3}, 1); + CartesianProductCombination combination = new CartesianProductCombination(new int[]{2, 3}, 1); combination.firstTaskWithFixedChunk(1); - verifyCombination(combination, new int[]{0,1}, 1); + verifyCombination(combination, new int[]{0, 1}, 1); assertTrue(combination.nextTaskWithFixedChunk()); - verifyCombination(combination, new int[]{1,1}, 4); + verifyCombination(combination, new int[]{1, 1}, 4); assertFalse(combination.nextTaskWithFixedChunk()); } private void testCombinationThreeWay() { - CartesianProductCombination combination = new CartesianProductCombination(new int[]{2,2,2}, 1); + CartesianProductCombination combination = new CartesianProductCombination(new int[]{2, 2, 2}, 1); combination.firstTaskWithFixedChunk(1); - verifyCombination(combination, new int[]{0,1,0}, 2); + verifyCombination(combination, new int[]{0, 1, 0}, 2); assertTrue(combination.nextTaskWithFixedChunk()); - verifyCombination(combination, new int[]{0,1,1}, 3); + verifyCombination(combination, new int[]{0, 1, 1}, 3); assertTrue(combination.nextTaskWithFixedChunk()); - verifyCombination(combination, new int[]{1,1,0}, 6); + verifyCombination(combination, new int[]{1, 1, 0}, 6); assertTrue(combination.nextTaskWithFixedChunk()); - verifyCombination(combination, new int[]{1,1,1}, 7); + verifyCombination(combination, new int[]{1, 1, 1}, 7); assertFalse(combination.nextTaskWithFixedChunk()); } @@ -82,7 +82,7 @@ public void testCombinationWithFixedPartition() { @Test(timeout = 5000) public void testCombination() { - CartesianProductCombination combination = new CartesianProductCombination(new int[]{2,3}); + CartesianProductCombination combination = new CartesianProductCombination(new int[]{2, 3}); List list = combination.getCombination(); for (int i = 0; i < 2; i++) { for (int j = 0; j < 3; j++) { @@ -101,19 +101,20 @@ public void testCombination() { @Test(timeout = 5000) public void testFromTaskId() { for (int i = 0; i < 6; i++) { - List list = CartesianProductCombination.fromTaskId(new int[]{2,3}, i) - .getCombination(); - assertTrue(list.get(0) == i/3); - assertTrue(list.get(1) == i%3); + List list = CartesianProductCombination.fromTaskId(new int[]{2, 3}, i) + .getCombination(); + assertTrue(list.get(0) == i / 3); + assertTrue(list.get(1) == i % 3); } } @Test(timeout = 5000) public void testRejectZero() { - int[] numChunk = new int[] {0 ,1}; + int[] numChunk = new int[]{0, 1}; try { new CartesianProductCombination(numChunk); assertTrue(false); - } catch (Exception ignored) {} + } catch (Exception ignored) { + } } } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductConfig.java index 3b4200a628..2ee2b1ce73 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductConfig.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductConfig.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -55,10 +55,10 @@ public void testSerializationPartitioned() throws IOException { byte[] bytes = new byte[10]; (new Random()).nextBytes(bytes); CartesianProductFilterDescriptor filterDescriptor = - new CartesianProductFilterDescriptor(filterClassName) - .setUserPayload(UserPayload.create(ByteBuffer.wrap(bytes))); + new CartesianProductFilterDescriptor(filterClassName) + .setUserPayload(UserPayload.create(ByteBuffer.wrap(bytes))); CartesianProductConfig config = - new CartesianProductConfig(vertexPartitionMap, filterDescriptor); + new CartesianProductConfig(vertexPartitionMap, filterDescriptor); UserPayload payload = config.toUserPayload(conf); CartesianProductConfig parsedConfig = CartesianProductConfig.fromUserPayload(payload); assertConfigEquals(config, parsedConfig); @@ -71,14 +71,14 @@ public void testSerializationFair() throws Exception { sourceVertices.add("v2"); sourceVertices.add("v3"); CartesianProductConfig config = - new CartesianProductConfig(sourceVertices); + new CartesianProductConfig(sourceVertices); UserPayload payload = config.toUserPayload(conf); CartesianProductConfig parsedConfig = CartesianProductConfig.fromUserPayload(payload); assertConfigEquals(config, parsedConfig); // fair cartesian product config should have null in numPartitions fields try { - config = new CartesianProductConfig(false, new int[]{}, new String[]{"v0","v1"},null); + config = new CartesianProductConfig(false, new int[]{}, new String[]{"v0", "v1"}, null); config.checkNumPartitions(); } catch (Exception e) { return; @@ -88,12 +88,12 @@ public void testSerializationFair() throws Exception { private void assertConfigEquals(CartesianProductConfig config1, CartesianProductConfig config2) { assertArrayEquals(config1.getSourceVertices().toArray(new String[0]), - config2.getSourceVertices().toArray(new String[0])); + config2.getSourceVertices().toArray(new String[0])); if (config1.getNumPartitions() == null) { assertNull(config2.getNumPartitions()); } else { assertArrayEquals(Ints.toArray(config1.getNumPartitions()), - Ints.toArray(config2.getNumPartitions())); + Ints.toArray(config2.getNumPartitions())); } CartesianProductFilterDescriptor descriptor1 = config1.getFilterDescriptor(); CartesianProductFilterDescriptor descriptor2 = config1.getFilterDescriptor(); @@ -121,11 +121,11 @@ public void testFairCartesianProductConfig() { // conf not set CartesianProductConfigProto proto = config.toProto(conf); assertEquals(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MAX_PARALLELISM_DEFAULT, - proto.getMaxParallelism()); + proto.getMaxParallelism()); assertEquals(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_MIN_OPS_PER_WORKER_DEFAULT, - proto.getMinOpsPerWorker()); + proto.getMinOpsPerWorker()); assertEquals(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_ENABLE_GROUPING_DEFAULT, - proto.getEnableGrouping()); + proto.getEnableGrouping()); assertFalse(proto.hasNumPartitionsForFairCase()); assertFalse(proto.hasGroupingFraction()); @@ -142,4 +142,4 @@ public void testFairCartesianProductConfig() { assertEquals(0.75f, proto.getGroupingFraction(), 0.01); assertEquals(25, proto.getNumPartitionsForFairCase()); } -} \ No newline at end of file +} diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductEdgeManager.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductEdgeManager.java index 58f460fc41..531e5f11a3 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductEdgeManager.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductEdgeManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -40,26 +40,26 @@ public void testInitialize() throws Exception { // partitioned case CartesianProductConfigProto.Builder builder = CartesianProductConfigProto.newBuilder(); builder.setIsPartitioned(true) - .addAllSources(Arrays.asList("v0", "v1")) - .addAllNumPartitions(Ints.asList(2,3)) - .setMaxParallelism(100).setMinOpsPerWorker(1); + .addAllSources(Arrays.asList("v0", "v1")) + .addAllNumPartitions(Ints.asList(2, 3)) + .setMaxParallelism(100).setMinOpsPerWorker(1); UserPayload payload = UserPayload.create(ByteBuffer.wrap(builder.build().toByteArray())); when(context.getUserPayload()).thenReturn(payload); edgeManager.initialize(); assertTrue(edgeManager.getEdgeManagerReal() - instanceof CartesianProductEdgeManagerPartitioned); + instanceof CartesianProductEdgeManagerPartitioned); // unpartitioned case builder.clear(); builder.setIsPartitioned(false) - .addAllSources(Arrays.asList("v0", "v1")) - .addAllNumChunks(Ints.asList(2,3)) - .setMaxParallelism(100).setMinOpsPerWorker(1); + .addAllSources(Arrays.asList("v0", "v1")) + .addAllNumChunks(Ints.asList(2, 3)) + .setMaxParallelism(100).setMinOpsPerWorker(1); payload = UserPayload.create(ByteBuffer.wrap(builder.build().toByteArray())); when(context.getUserPayload()).thenReturn(payload); when(context.getSourceVertexNumTasks()).thenReturn(2); edgeManager.initialize(); assertTrue(edgeManager.getEdgeManagerReal() - instanceof FairCartesianProductEdgeManager); + instanceof FairCartesianProductEdgeManager); } } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductEdgeManagerPartitioned.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductEdgeManagerPartitioned.java index 462760fd03..a75194b220 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductEdgeManagerPartitioned.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductEdgeManagerPartitioned.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -54,7 +54,7 @@ public void setup() { public void testTwoWay() throws Exception { CartesianProductConfigProto.Builder builder = CartesianProductConfigProto.newBuilder(); builder.setIsPartitioned(true).addSources("v0").addSources("v1") - .addNumPartitions(3).addNumPartitions(4); + .addNumPartitions(3).addNumPartitions(4); when(mockContext.getDestinationVertexNumTasks()).thenReturn(12); CartesianProductConfigProto config = builder.build(); testTwoWayV0(config); @@ -67,18 +67,18 @@ private void testTwoWayV0(CartesianProductConfigProto config) throws Exception { edgeManager.initialize(config); CompositeEventRouteMetadata compositeRoutingData = - edgeManager.routeCompositeDataMovementEventToDestination(1, 1); + edgeManager.routeCompositeDataMovementEventToDestination(1, 1); assertNotNull(compositeRoutingData); assertEquals(1, compositeRoutingData.getCount()); assertEquals(0, compositeRoutingData.getSource()); assertEquals(1, compositeRoutingData.getTarget()); - EventRouteMetadata routingData = edgeManager.routeDataMovementEventToDestination(1,0,1); + EventRouteMetadata routingData = edgeManager.routeDataMovementEventToDestination(1, 0, 1); assertNotNull(routingData); assertEquals(1, routingData.getNumEvents()); assertArrayEquals(new int[]{1}, routingData.getTargetIndices()); - routingData = edgeManager.routeDataMovementEventToDestination(1,1,1); + routingData = edgeManager.routeDataMovementEventToDestination(1, 1, 1); assertNull(routingData); routingData = edgeManager.routeInputSourceTaskFailedEventToDestination(1, 1); @@ -99,7 +99,7 @@ private void testTwoWayV1(CartesianProductConfigProto config) throws Exception { edgeManager.initialize(config); CompositeEventRouteMetadata compositeRoutingData = - edgeManager.routeCompositeDataMovementEventToDestination(1, 1); + edgeManager.routeCompositeDataMovementEventToDestination(1, 1); assertNotNull(compositeRoutingData); assertEquals(1, compositeRoutingData.getCount()); assertEquals(1, compositeRoutingData.getSource()); @@ -148,8 +148,8 @@ public void testTwoWayWithFilter() throws Exception { ByteBuffer buffer = ByteBuffer.allocate(2).putChar('>'); buffer.flip(); builder.setIsPartitioned(true).addSources("v0").addSources("v1") - .addNumPartitions(3).addNumPartitions(4).setFilterClassName(TestFilter.class.getName()) - .setFilterUserPayload(ByteString.copyFrom(buffer)); + .addNumPartitions(3).addNumPartitions(4).setFilterClassName(TestFilter.class.getName()) + .setFilterUserPayload(ByteString.copyFrom(buffer)); CartesianProductConfigProto config = builder.build(); when(mockContext.getDestinationVertexNumTasks()).thenReturn(3); testTwoWayV0WithFilter(config); @@ -162,7 +162,7 @@ private void testTwoWayV0WithFilter(CartesianProductConfigProto config) throws E edgeManager.initialize(config); CompositeEventRouteMetadata compositeRoutingData = - edgeManager.routeCompositeDataMovementEventToDestination(1, 1); + edgeManager.routeCompositeDataMovementEventToDestination(1, 1); assertNotNull(compositeRoutingData); assertEquals(1, compositeRoutingData.getCount()); assertEquals(2, compositeRoutingData.getSource()); @@ -186,7 +186,7 @@ private void testTwoWayV1WithFilter(CartesianProductConfigProto config) throws E edgeManager.initialize(config); CompositeEventRouteMetadata compositeRoutingData = - edgeManager.routeCompositeDataMovementEventToDestination(1, 1); + edgeManager.routeCompositeDataMovementEventToDestination(1, 1); assertNotNull(compositeRoutingData); assertEquals(1, compositeRoutingData.getCount()); assertEquals(0, compositeRoutingData.getSource()); @@ -213,7 +213,7 @@ private void testTwoWayV1WithFilter(CartesianProductConfigProto config) throws E public void testThreeWay() throws Exception { CartesianProductConfigProto.Builder builder = CartesianProductConfigProto.newBuilder(); builder.setIsPartitioned(true).addSources("v0").addSources("v1").addSources("v2") - .addNumPartitions(4).addNumPartitions(3).addNumPartitions(2); + .addNumPartitions(4).addNumPartitions(3).addNumPartitions(2); CartesianProductConfigProto config = builder.build(); when(mockContext.getDestinationVertexNumTasks()).thenReturn(24); @@ -229,7 +229,7 @@ private void testThreeWayV0(CartesianProductConfigProto config) throws Exception edgeManager.initialize(config); CompositeEventRouteMetadata compositeRoutingData = - edgeManager.routeCompositeDataMovementEventToDestination(1, 1); + edgeManager.routeCompositeDataMovementEventToDestination(1, 1); assertNotNull(compositeRoutingData); assertEquals(1, compositeRoutingData.getCount()); assertEquals(0, compositeRoutingData.getSource()); @@ -253,7 +253,7 @@ private void testThreeWayV1(CartesianProductConfigProto config) throws Exception edgeManager.initialize(config); CompositeEventRouteMetadata compositeRoutingData = - edgeManager.routeCompositeDataMovementEventToDestination(1, 1); + edgeManager.routeCompositeDataMovementEventToDestination(1, 1); assertNotNull(compositeRoutingData); assertEquals(1, compositeRoutingData.getCount()); assertEquals(0, compositeRoutingData.getSource()); @@ -277,7 +277,7 @@ private void testThreeWayV2(CartesianProductConfigProto config) throws Exception edgeManager.initialize(config); CompositeEventRouteMetadata compositeRoutingData = - edgeManager.routeCompositeDataMovementEventToDestination(1, 1); + edgeManager.routeCompositeDataMovementEventToDestination(1, 1); assertNotNull(compositeRoutingData); assertEquals(1, compositeRoutingData.getCount()); assertEquals(1, compositeRoutingData.getSource()); diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductVertexManager.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductVertexManager.java index 5846c8b526..1360215b79 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductVertexManager.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductVertexManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -48,11 +48,11 @@ public class TestCartesianProductVertexManager { private CartesianProductConfig config; private Map edgePropertyMap; private EdgeProperty cpEdge = EdgeProperty.create(EdgeManagerPluginDescriptor.create( - CartesianProductEdgeManager.class.getName()), null, null, null, null); + CartesianProductEdgeManager.class.getName()), null, null, null, null); private EdgeProperty customEdge = EdgeProperty.create(EdgeManagerPluginDescriptor.create( - "OTHER_EDGE"), null, null, null, null); + "OTHER_EDGE"), null, null, null, null); private EdgeProperty broadcastEdge = - EdgeProperty.create(DataMovementType.BROADCAST, null, null, null, null); + EdgeProperty.create(DataMovementType.BROADCAST, null, null, null, null); @Before public void setup() { @@ -79,7 +79,8 @@ public void testRejectPredefinedParallelism() throws Exception { try { vertexManager = new CartesianProductVertexManager(context); assertTrue(false); - } catch (Exception ignored){} + } catch (Exception ignored) { + } } @Test(timeout = 5000) @@ -88,7 +89,7 @@ public void testChooseRealVertexManager() throws Exception { config = new CartesianProductConfig(new int[]{2, 3}, new String[]{"v0", "v1"}, null); vertexManager.initialize(); assertTrue(vertexManager.getVertexManagerReal() - instanceof CartesianProductVertexManagerPartitioned); + instanceof CartesianProductVertexManagerPartitioned); // unpartitioned case List sourceVertices = new ArrayList<>(); @@ -97,7 +98,7 @@ public void testChooseRealVertexManager() throws Exception { config = new CartesianProductConfig(sourceVertices); vertexManager.initialize(); assertTrue(vertexManager.getVertexManagerReal() - instanceof FairCartesianProductVertexManager); + instanceof FairCartesianProductVertexManager); } @Test(timeout = 5000) @@ -112,7 +113,8 @@ public void testCheckDAGConfigConsistent() throws Exception { try { vertexManager.initialize(); assertTrue(false); - } catch (Exception ignored) {} + } catch (Exception ignored) { + } // non-cartesian-product edge in dag but in config edgePropertyMap.put("v2", broadcastEdge); @@ -120,20 +122,23 @@ public void testCheckDAGConfigConsistent() throws Exception { try { vertexManager.initialize(); assertTrue(false); - } catch (Exception ignored) {} + } catch (Exception ignored) { + } edgePropertyMap.put("v2", customEdge); try { vertexManager.initialize(); assertTrue(false); - } catch (Exception ignored) {} + } catch (Exception ignored) { + } // edge in config but not in dag edgePropertyMap.remove("v2"); try { vertexManager.initialize(); assertTrue(false); - } catch (Exception ignored) {} + } catch (Exception ignored) { + } } @Test(timeout = 5000) @@ -151,7 +156,8 @@ public void testCheckDAGConfigConsistentWithVertexGroup() throws Exception { try { vertexManager.initialize(); assertTrue(false); - } catch (Exception ignored) {} + } catch (Exception ignored) { + } } @Test(timeout = 5000) @@ -162,7 +168,8 @@ public void testOtherEdgeType() throws Exception { try { vertexManager.initialize(); assertTrue(false); - } catch (Exception ignored) {} + } catch (Exception ignored) { + } // broadcast edge should be allowed and other non-custom edge shouldn't be allowed for (DataMovementType type : DataMovementType.values()) { diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductVertexManagerPartitioned.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductVertexManagerPartitioned.java index a77e2c712e..3339d5e294 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductVertexManagerPartitioned.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestCartesianProductVertexManagerPartitioned.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -68,12 +68,12 @@ public class TestCartesianProductVertexManagerPartitioned { public void setup() throws TezReflectionException { CartesianProductConfigProto.Builder builder = CartesianProductConfigProto.newBuilder(); builder.setIsPartitioned(true).addSources("v0").addSources("v1") - .addNumPartitions(2).addNumPartitions(2); + .addNumPartitions(2).addNumPartitions(2); setupWithConfig(builder.build()); } private void setupWithConfig(CartesianProductConfigProto config) - throws TezReflectionException { + throws TezReflectionException { MockitoAnnotations.openMocks(this); context = mock(VertexManagerPluginContext.class); when(context.getVertexName()).thenReturn("cp"); @@ -81,9 +81,9 @@ private void setupWithConfig(CartesianProductConfigProto config) vertexManager = new CartesianProductVertexManagerPartitioned(context); Map edgePropertyMap = new HashMap<>(); edgePropertyMap.put("v0", EdgeProperty.create(EdgeManagerPluginDescriptor.create( - CartesianProductEdgeManager.class.getName()), null, null, null, null)); + CartesianProductEdgeManager.class.getName()), null, null, null, null)); edgePropertyMap.put("v1", EdgeProperty.create(EdgeManagerPluginDescriptor.create( - CartesianProductEdgeManager.class.getName()), null, null, null, null)); + CartesianProductEdgeManager.class.getName()), null, null, null, null)); edgePropertyMap.put("v2", EdgeProperty.create(BROADCAST, null, null, null, null)); when(context.getInputVertexEdgeProperties()).thenReturn(edgePropertyMap); when(context.getVertexNumTasks(eq("v0"))).thenReturn(4); @@ -95,22 +95,22 @@ private void setupWithConfig(CartesianProductConfigProto config) for (int i = 0; i < 3; i++) { for (int j = 0; j < 4; j++) { allCompletions.add(new TaskAttemptIdentifierImpl("dag", "v" + i, - TezTaskAttemptID.getInstance(TezTaskID.getInstance(TezVertexID.getInstance( - TezDAGID.getInstance("0", 0, 0), i), j), 0))); + TezTaskAttemptID.getInstance(TezTaskID.getInstance(TezVertexID.getInstance( + TezDAGID.getInstance("0", 0, 0), i), j), 0))); } } } private void testReconfigureVertexHelper(CartesianProductConfigProto config, int parallelism) - throws Exception { + throws Exception { setupWithConfig(config); ArgumentCaptor parallelismCaptor = ArgumentCaptor.forClass(Integer.class); vertexManager.onVertexStateUpdated(new VertexStateUpdate("v0", VertexState.CONFIGURED)); verify(context, times(1)).reconfigureVertex(parallelismCaptor.capture(), isNull(), edgePropertiesCaptor.capture()); - assertEquals((int)parallelismCaptor.getValue(), parallelism); + assertEquals((int) parallelismCaptor.getValue(), parallelism); assertNull(edgePropertiesCaptor.getValue()); } @@ -118,7 +118,7 @@ private void testReconfigureVertexHelper(CartesianProductConfigProto config, public void testReconfigureVertex() throws Exception { CartesianProductConfigProto.Builder builder = CartesianProductConfigProto.newBuilder(); builder.setIsPartitioned(true).addSources("v0").addSources("v1") - .addNumPartitions(5).addNumPartitions(5).setFilterClassName(TestFilter.class.getName()); + .addNumPartitions(5).addNumPartitions(5).setFilterClassName(TestFilter.class.getName()); testReconfigureVertexHelper(builder.build(), 10); builder.clearFilterClassName(); testReconfigureVertexHelper(builder.build(), 25); @@ -130,7 +130,6 @@ public void testScheduling() throws Exception { vertexManager.onVertexStateUpdated(new VertexStateUpdate("v0", VertexState.CONFIGURED)); vertexManager.onVertexStateUpdated(new VertexStateUpdate("v1", VertexState.CONFIGURED)); - vertexManager.onSourceTaskCompleted(allCompletions.get(0)); vertexManager.onSourceTaskCompleted(allCompletions.get(1)); verify(context, never()).scheduleTasks(any()); @@ -152,10 +151,10 @@ public void testScheduling() throws Exception { for (int i = 3; i < 6; i++) { vertexManager.onSourceTaskCompleted(allCompletions.get(i)); - verify(context, times(i-1)).scheduleTasks(scheduleTaskRequestCaptor.capture()); + verify(context, times(i - 1)).scheduleTasks(scheduleTaskRequestCaptor.capture()); scheduleTaskRequests = scheduleTaskRequestCaptor.getValue(); assertEquals(1, scheduleTaskRequests.size()); - assertEquals(i-2, scheduleTaskRequests.get(0).getTaskIndex()); + assertEquals(i - 2, scheduleTaskRequests.get(0).getTaskIndex()); } for (int i = 6; i < 8; i++) { diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestFairCartesianProductEdgeManager.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestFairCartesianProductEdgeManager.java index ac7262ee01..85d763c401 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestFairCartesianProductEdgeManager.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestFairCartesianProductEdgeManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -74,7 +74,7 @@ private void testEdgeManager(CartesianProductConfigProto conf, String vName, int TestData srcFailInvalid, TestData srcFailValid, TestData inputError, TestData numDestInput, TestData numSrcOutputTest, TestData numConsumerTest) - throws Exception { + throws Exception { when(mockContext.getSourceVertexName()).thenReturn(vName); when(mockContext.getSourceVertexNumTasks()).thenReturn(numTask); when(mockContext.getVertexGroupName()).thenReturn(groupName); @@ -84,14 +84,14 @@ private void testEdgeManager(CartesianProductConfigProto conf, String vName, int if (cDMEInvalid != null) { cDME = edgeManager.routeCompositeDataMovementEventToDestination(cDMEInvalid.srcId, - cDMEInvalid.destId); + cDMEInvalid.destId); assertNull(cDME); } cDME = edgeManager.routeCompositeDataMovementEventToDestination(cDMEValid.srcId, - cDMEValid.destId); + cDMEValid.destId); assertNotNull(cDME); - CompositeEventRouteMetadata expectedCDME = (CompositeEventRouteMetadata)(cDMEValid.expected); + CompositeEventRouteMetadata expectedCDME = (CompositeEventRouteMetadata) (cDMEValid.expected); assertEquals(expectedCDME.getCount(), cDME.getCount()); assertEquals(expectedCDME.getTarget(), cDME.getTarget()); assertEquals(expectedCDME.getSource(), cDME.getSource()); @@ -99,26 +99,26 @@ private void testEdgeManager(CartesianProductConfigProto conf, String vName, int EventRouteMetadata dme; if (srcFailInvalid != null) { dme = edgeManager.routeInputSourceTaskFailedEventToDestination(srcFailInvalid.srcId, - srcFailInvalid.destId); + srcFailInvalid.destId); assertNull(dme); } dme = edgeManager.routeInputSourceTaskFailedEventToDestination(srcFailValid.srcId, - srcFailValid.destId); + srcFailValid.destId); assertNotNull(dme); - EventRouteMetadata expectedDME = (EventRouteMetadata)(srcFailValid.expected); + EventRouteMetadata expectedDME = (EventRouteMetadata) (srcFailValid.expected); assertEquals(expectedDME.getNumEvents(), dme.getNumEvents()); assertArrayEquals(expectedDME.getTargetIndices(), dme.getTargetIndices()); assertEquals(inputError.expected, - edgeManager.routeInputErrorEventToSource(inputError.destId, inputError.inputId)); + edgeManager.routeInputErrorEventToSource(inputError.destId, inputError.inputId)); assertEquals(numDestInput.expected, - edgeManager.getNumDestinationTaskPhysicalInputs(numDestInput.destId)); + edgeManager.getNumDestinationTaskPhysicalInputs(numDestInput.destId)); assertEquals(numSrcOutputTest.expected, - edgeManager.getNumSourceTaskPhysicalOutputs(numSrcOutputTest.srcId)); + edgeManager.getNumSourceTaskPhysicalOutputs(numSrcOutputTest.srcId)); assertEquals(numConsumerTest.expected, - edgeManager.getNumDestinationConsumerTasks(numConsumerTest.srcId)); + edgeManager.getNumDestinationConsumerTasks(numConsumerTest.srcId)); } /** @@ -129,18 +129,18 @@ private void testEdgeManager(CartesianProductConfigProto conf, String vName, int public void testTwoWayAllVertex() throws Exception { CartesianProductConfigProto.Builder builder = CartesianProductConfigProto.newBuilder(); builder.setIsPartitioned(false).addSources("v0").addSources("v1") - .addNumChunks(2).addNumChunks(3).setMaxParallelism(10).setNumPartitionsForFairCase(10); + .addNumChunks(2).addNumChunks(3).setMaxParallelism(10).setNumPartitionsForFairCase(10); CartesianProductConfigProto config = builder.build(); testEdgeManager(config, "v0", 2, null, dataForRouting(1, 1, null), - dataForRouting(1, 3, CompositeEventRouteMetadata.create(10, 0, 0)), - dataForRouting(1, 1, null), - dataForRouting(1, 3, EventRouteMetadata.create(10, new int[]{0,1,2,3,4,5,6,7,8,9})), - dataForInputError(1, 0, 0), dataForDest(1, 10), dataForSrc(1, 10), dataForSrc(1, 3)); + dataForRouting(1, 3, CompositeEventRouteMetadata.create(10, 0, 0)), + dataForRouting(1, 1, null), + dataForRouting(1, 3, EventRouteMetadata.create(10, new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})), + dataForInputError(1, 0, 0), dataForDest(1, 10), dataForSrc(1, 10), dataForSrc(1, 3)); testEdgeManager(config, "v1", 30, null, dataForRouting(1, 2, null), - dataForRouting(1, 0, CompositeEventRouteMetadata.create(10, 10, 0)), - dataForRouting(1, 2, null), - dataForRouting(1, 0, EventRouteMetadata.create(10, new int[]{10,11,12,13,14,15,16,17,18,19})), - dataForInputError(1,0,10), dataForDest(1, 100), dataForSrc(1, 10), dataForSrc(1, 2)); + dataForRouting(1, 0, CompositeEventRouteMetadata.create(10, 10, 0)), + dataForRouting(1, 2, null), + dataForRouting(1, 0, EventRouteMetadata.create(10, new int[]{10, 11, 12, 13, 14, 15, 16, 17, 18, 19})), + dataForInputError(1, 0, 10), dataForDest(1, 100), dataForSrc(1, 10), dataForSrc(1, 2)); } /** @@ -152,24 +152,24 @@ public void testTwoWayAllVertex() throws Exception { public void testThreeWayAllVertex() throws Exception { CartesianProductConfigProto.Builder builder = CartesianProductConfigProto.newBuilder(); builder.setIsPartitioned(false).addSources("v0").addSources("v1").addSources("v2") - .addNumChunks(2).addNumChunks(3).addNumChunks(4) - .setMaxParallelism(12).setNumPartitionsForFairCase(12); + .addNumChunks(2).addNumChunks(3).addNumChunks(4) + .setMaxParallelism(12).setNumPartitionsForFairCase(12); CartesianProductConfigProto config = builder.build(); testEdgeManager(config, "v0", 2, null, dataForRouting(1, 1, null), - dataForRouting(1, 12, CompositeEventRouteMetadata.create(12, 0, 0)), - dataForRouting(1, 1, null), - dataForRouting(1, 12, EventRouteMetadata.create(12, new int[]{0,1,2,3,4,5,6,7,8,9,10,11})), - dataForInputError(1, 0, 0), dataForDest(1, 12), dataForSrc(1, 12), dataForSrc(1, 12)); + dataForRouting(1, 12, CompositeEventRouteMetadata.create(12, 0, 0)), + dataForRouting(1, 1, null), + dataForRouting(1, 12, EventRouteMetadata.create(12, new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11})), + dataForInputError(1, 0, 0), dataForDest(1, 12), dataForSrc(1, 12), dataForSrc(1, 12)); testEdgeManager(config, "v1", 30, null, dataForRouting(1, 4, null), - dataForRouting(1, 13, CompositeEventRouteMetadata.create(12, 12, 0)), - dataForRouting(1, 4, null), - dataForRouting(1, 13, - EventRouteMetadata.create(12, new int[]{12,13,14,15,16,17,18,19,20,21,22,23})), - dataForInputError(1, 0, 0), dataForDest(1, 120), dataForSrc(1, 12), dataForSrc(1, 8)); + dataForRouting(1, 13, CompositeEventRouteMetadata.create(12, 12, 0)), + dataForRouting(1, 4, null), + dataForRouting(1, 13, + EventRouteMetadata.create(12, new int[]{12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23})), + dataForInputError(1, 0, 0), dataForDest(1, 120), dataForSrc(1, 12), dataForSrc(1, 8)); testEdgeManager(config, "v2", 1, null, - null, dataForRouting(0, 13, CompositeEventRouteMetadata.create(3, 0, 3)), - null, dataForRouting(0, 13, EventRouteMetadata.create(3, new int[]{0,1,2})), - dataForInputError(1, 0, 0), dataForDest(1, 3), dataForSrc(0, 12), dataForSrc(0, 24)); + null, dataForRouting(0, 13, CompositeEventRouteMetadata.create(3, 0, 3)), + null, dataForRouting(0, 13, EventRouteMetadata.create(3, new int[]{0, 1, 2})), + dataForInputError(1, 0, 0), dataForDest(1, 3), dataForSrc(0, 12), dataForSrc(0, 24)); } /** @@ -183,19 +183,19 @@ null, dataForRouting(0, 13, EventRouteMetadata.create(3, new int[]{0,1,2})), public void testTwoWayVertexWithVertexGroup() throws Exception { CartesianProductConfigProto.Builder builder = CartesianProductConfigProto.newBuilder(); builder.setIsPartitioned(false).addSources("v0").addSources("g0") - .addNumChunks(2).addNumChunks(3).setPositionInGroup(10).setNumPartitionsForFairCase(10) - .addNumTaskPerVertexInGroup(10).addNumTaskPerVertexInGroup(20).setPositionInGroup(0); + .addNumChunks(2).addNumChunks(3).setPositionInGroup(10).setNumPartitionsForFairCase(10) + .addNumTaskPerVertexInGroup(10).addNumTaskPerVertexInGroup(20).setPositionInGroup(0); testEdgeManager(builder.build(), "v1", 10, "g0", dataForRouting(0, 4, null), - dataForRouting(0, 3, CompositeEventRouteMetadata.create(10, 0, 0)), - dataForRouting(0, 4, null), - dataForRouting(0, 3, EventRouteMetadata.create(10, new int[]{0,1,2,3,4,5,6,7,8,9})), - dataForInputError(3, 0, 0), dataForDest(2, 34), dataForSrc(0, 10), dataForSrc(0, 2)); + dataForRouting(0, 3, CompositeEventRouteMetadata.create(10, 0, 0)), + dataForRouting(0, 4, null), + dataForRouting(0, 3, EventRouteMetadata.create(10, new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})), + dataForInputError(3, 0, 0), dataForDest(2, 34), dataForSrc(0, 10), dataForSrc(0, 2)); builder.setPositionInGroup(1); testEdgeManager(builder.build(), "v2", 20, "g0", dataForRouting(1, 1, null), - dataForRouting(6, 1, CompositeEventRouteMetadata.create(4, 33, 6)), - dataForRouting(1, 1, null), - dataForRouting(6, 1, EventRouteMetadata.create(4, new int[]{33,34,35,36})), - dataForInputError(1, 33, 6), dataForDest(0, 66), dataForSrc(1, 10), dataForSrc(6, 4)); + dataForRouting(6, 1, CompositeEventRouteMetadata.create(4, 33, 6)), + dataForRouting(1, 1, null), + dataForRouting(6, 1, EventRouteMetadata.create(4, new int[]{33, 34, 35, 36})), + dataForInputError(1, 33, 6), dataForDest(0, 66), dataForSrc(1, 10), dataForSrc(6, 4)); } /** @@ -210,19 +210,19 @@ public void testTwoWayVertexWithVertexGroup() throws Exception { public void testTwoWayAllVertexGroup() throws Exception { CartesianProductConfigProto.Builder builder = CartesianProductConfigProto.newBuilder(); builder.setIsPartitioned(false).addSources("g0").addSources("g1") - .addNumChunks(2).addNumChunks(3).setMaxParallelism(10).setNumPartitionsForFairCase(10) - .addNumTaskPerVertexInGroup(2).addNumTaskPerVertexInGroup(5).setPositionInGroup(0); + .addNumChunks(2).addNumChunks(3).setMaxParallelism(10).setNumPartitionsForFairCase(10) + .addNumTaskPerVertexInGroup(2).addNumTaskPerVertexInGroup(5).setPositionInGroup(0); testEdgeManager(builder.build(), "v0", 2, "g0", dataForRouting(1, 1, null), - dataForRouting(0, 1, CompositeEventRouteMetadata.create(10, 0, 0)), - dataForRouting(1, 1, null), - dataForRouting(0, 1, EventRouteMetadata.create(10, new int[]{0,1,2,3,4,5,6,7,8,9})), - dataForInputError(1, 0, 0), dataForDest(1, 10), dataForSrc(1, 10), dataForSrc(1, 3)); + dataForRouting(0, 1, CompositeEventRouteMetadata.create(10, 0, 0)), + dataForRouting(1, 1, null), + dataForRouting(0, 1, EventRouteMetadata.create(10, new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})), + dataForInputError(1, 0, 0), dataForDest(1, 10), dataForSrc(1, 10), dataForSrc(1, 3)); builder.setPositionInGroup(1); testEdgeManager(builder.build(), "v1", 5, "g0", dataForRouting(3, 1, null), - dataForRouting(1, 1, CompositeEventRouteMetadata.create(10, 20, 0)), - dataForRouting(3, 1, null), - dataForRouting(1, 1, EventRouteMetadata.create(10, new int[]{20,21,22,23,24,25,26,27,28,29})), - dataForInputError(1, 15, 0), dataForDest(1, 25), dataForSrc(1, 10), dataForSrc(1, 3)); + dataForRouting(1, 1, CompositeEventRouteMetadata.create(10, 20, 0)), + dataForRouting(3, 1, null), + dataForRouting(1, 1, EventRouteMetadata.create(10, new int[]{20, 21, 22, 23, 24, 25, 26, 27, 28, 29})), + dataForInputError(1, 15, 0), dataForDest(1, 25), dataForSrc(1, 10), dataForSrc(1, 3)); } @Test(timeout = 5000) @@ -242,4 +242,4 @@ public void testNumPartition() throws Exception { edgeManager.initialize(builder.build()); assertEquals(20, edgeManager.getNumSourceTaskPhysicalOutputs(0)); } -} \ No newline at end of file +} diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestFairCartesianProductVertexManager.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestFairCartesianProductVertexManager.java index 54d771ad08..7e3d48100e 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestFairCartesianProductVertexManager.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestFairCartesianProductVertexManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -86,8 +86,8 @@ private void setupDAGVertexOnly(int maxParallelism, long minOpsPerWorker, int nu CartesianProductConfigProto.Builder builder = CartesianProductConfigProto.newBuilder(); builder.setIsPartitioned(false).addSources("v0").addSources("v1") - .setMaxParallelism(maxParallelism).setMinOpsPerWorker(minOpsPerWorker) - .setNumPartitionsForFairCase(numPartition); + .setMaxParallelism(maxParallelism).setMinOpsPerWorker(minOpsPerWorker) + .setNumPartitionsForFairCase(numPartition); vertexManager.initialize(builder.build()); } @@ -103,8 +103,8 @@ private void setupDAGVertexOnlyWithBroadcast(int maxParallelism, long minWorkloa CartesianProductConfigProto.Builder builder = CartesianProductConfigProto.newBuilder(); builder.setIsPartitioned(false).addSources("v0").addSources("v1") - .setMaxParallelism(maxParallelism).setMinOpsPerWorker(minWorkloadPerWorker) - .setNumPartitionsForFairCase(maxParallelism); + .setMaxParallelism(maxParallelism).setMinOpsPerWorker(minWorkloadPerWorker) + .setNumPartitionsForFairCase(maxParallelism); vertexManager.initialize(builder.build()); } @@ -122,8 +122,8 @@ private void setupDAGVertexGroup(int maxParallelism, long minWorkloadPerWorker, CartesianProductConfigProto.Builder builder = CartesianProductConfigProto.newBuilder(); builder.setIsPartitioned(false).addSources("v0").addSources("g0") - .setNumPartitionsForFairCase(maxParallelism).setMaxParallelism(maxParallelism) - .setMinOpsPerWorker(minWorkloadPerWorker); + .setNumPartitionsForFairCase(maxParallelism).setMaxParallelism(maxParallelism) + .setMinOpsPerWorker(minWorkloadPerWorker); vertexManager.initialize(builder.build()); } @@ -142,16 +142,16 @@ private void setupDAGVertexGroupOnly(int maxParallelism, long minWorkloadPerWork CartesianProductConfigProto.Builder builder = CartesianProductConfigProto.newBuilder(); builder.setIsPartitioned(false).addSources("g0").addSources("g1") - .setNumPartitionsForFairCase(maxParallelism) - .setMaxParallelism(maxParallelism).setMinOpsPerWorker(minWorkloadPerWorker); + .setNumPartitionsForFairCase(maxParallelism) + .setMaxParallelism(maxParallelism).setMinOpsPerWorker(minWorkloadPerWorker); vertexManager.initialize(builder.build()); } private Map getEdgePropertyMap(int numSrcV) { Map edgePropertyMap = new HashMap<>(); for (int i = 0; i < numSrcV; i++) { - edgePropertyMap.put("v"+i, EdgeProperty.create(EdgeManagerPluginDescriptor.create( - CartesianProductEdgeManager.class.getName()), null, null, null, null)); + edgePropertyMap.put("v" + i, EdgeProperty.create(EdgeManagerPluginDescriptor.create( + CartesianProductEdgeManager.class.getName()), null, null, null, null)); } return edgePropertyMap; } @@ -159,15 +159,15 @@ private Map getEdgePropertyMap(int numSrcV) { private void setSrcParallelism(VertexManagerPluginContext ctx, int multiplier, int... numTasks) { int i = 0; for (int numTask : numTasks) { - when(ctx.getVertexNumTasks(eq("v"+i))).thenReturn(numTask * multiplier); + when(ctx.getVertexNumTasks(eq("v" + i))).thenReturn(numTask * multiplier); i++; } } private TaskAttemptIdentifier getTaId(String vertexName, int taskId) { return new TaskAttemptIdentifierImpl("dag", vertexName, - TezTaskAttemptID.getInstance(TezTaskID.getInstance(TezVertexID.getInstance( - TezDAGID.getInstance("0", 0, 0), 0), taskId), 0)); + TezTaskAttemptID.getInstance(TezTaskID.getInstance(TezVertexID.getInstance( + TezDAGID.getInstance("0", 0, 0), 0), taskId), 0)); } private VertexManagerEvent getVMEvent(long numRecord, String vName, int taskId) { @@ -175,16 +175,16 @@ private VertexManagerEvent getVMEvent(long numRecord, String vName, int taskId) VertexManagerEventPayloadProto.Builder builder = VertexManagerEventPayloadProto.newBuilder(); builder.setNumRecord(numRecord); VertexManagerEvent vmEvent = - VertexManagerEvent.create("cp vertex", builder.build().toByteString().asReadOnlyByteBuffer()); + VertexManagerEvent.create("cp vertex", builder.build().toByteString().asReadOnlyByteBuffer()); vmEvent.setProducerAttemptIdentifier(getTaId(vName, taskId)); return vmEvent; } private void verifyEdgeProperties(EdgeProperty edgeProperty, String[] sources, int[] numChunksPerSrc, int maxParallelism) - throws InvalidProtocolBufferException { + throws InvalidProtocolBufferException { CartesianProductConfigProto config = CartesianProductConfigProto.parseFrom(ByteString.copyFrom( - edgeProperty.getEdgeManagerDescriptor().getUserPayload().getPayload())); + edgeProperty.getEdgeManagerDescriptor().getUserPayload().getPayload())); assertArrayEquals(sources, config.getSourcesList().toArray()); assertArrayEquals(numChunksPerSrc, Ints.toArray(config.getNumChunksList())); assertEquals(maxParallelism, config.getMaxParallelism()); @@ -192,9 +192,9 @@ private void verifyEdgeProperties(EdgeProperty edgeProperty, String[] sources, private void verifyVertexGroupInfo(EdgeProperty edgeProperty, int positionInGroup, int... numTaskPerVertexInGroup) - throws InvalidProtocolBufferException { + throws InvalidProtocolBufferException { CartesianProductConfigProto config = CartesianProductConfigProto.parseFrom(ByteString.copyFrom( - edgeProperty.getEdgeManagerDescriptor().getUserPayload().getPayload())); + edgeProperty.getEdgeManagerDescriptor().getUserPayload().getPayload())); assertEquals(positionInGroup, config.getPositionInGroup()); int i = 0; for (int numTask : numTaskPerVertexInGroup) { @@ -223,7 +223,7 @@ public void testDAGVertexOnlyGroupByMaxParallelism() throws Exception { vertexManager.onVertexManagerEventReceived(getVMEvent(250, "v0", 0)); verify(ctx, never()).reconfigureVertex( - anyInt(), any(), anyMap()); + anyInt(), any(), anyMap()); vertexManager.onVertexManagerEventReceived(getVMEvent(200, "v1", 0)); verify(ctx, times(1)).reconfigureVertex( @@ -292,7 +292,7 @@ public void testDAGVertexGroup() throws Exception { Map edgeProperties = edgePropertiesCaptor.getValue(); for (int i = 0; i < 3; i++) { verifyEdgeProperties(edgeProperties.get("v" + i), new String[]{"v0", "g0"}, - new int[]{20, 5}, 100); + new int[]{20, 5}, 100); } vertexManager.onVertexStarted(null); @@ -326,7 +326,7 @@ public void testDAGVertexGroupOnly() throws Exception { Map edgeProperties = edgePropertiesCaptor.getValue(); for (int i = 0; i < 4; i++) { verifyEdgeProperties(edgeProperties.get("v" + i), new String[]{"g0", "g1"}, - new int[]{10, 10}, 100); + new int[]{10, 10}, 100); } verifyVertexGroupInfo(edgeProperties.get("v0"), 0); verifyVertexGroupInfo(edgeProperties.get("v1"), 1, 2); @@ -362,7 +362,6 @@ public void testSchedulingVertexOnlyWithBroadcast() throws Exception { verifyScheduleRequest(1, 0, 1, 6, 7); } - @Test(timeout = 5000) public void testOnVertexStart() throws Exception { setupDAGVertexOnly(6, 1, 6, 1); @@ -386,14 +385,14 @@ public void testZeroSrcTask() throws Exception { CartesianProductConfigProto.Builder builder = CartesianProductConfigProto.newBuilder(); builder.setIsPartitioned(false).addSources("v0").addSources("v1") - .addNumChunks(2).addNumChunks(3).setMaxParallelism(6); + .addNumChunks(2).addNumChunks(3).setMaxParallelism(6); CartesianProductConfigProto config = builder.build(); Map edgePropertyMap = new HashMap<>(); edgePropertyMap.put("v0", EdgeProperty.create(EdgeManagerPluginDescriptor.create( - CartesianProductEdgeManager.class.getName()), null, null, null, null)); + CartesianProductEdgeManager.class.getName()), null, null, null, null)); edgePropertyMap.put("v1", EdgeProperty.create(EdgeManagerPluginDescriptor.create( - CartesianProductEdgeManager.class.getName()), null, null, null, null)); + CartesianProductEdgeManager.class.getName()), null, null, null, null)); when(ctx.getInputVertexEdgeProperties()).thenReturn(edgePropertyMap); vertexManager.initialize(config); @@ -410,13 +409,14 @@ private void setupGroupingFractionTest() throws Exception { CartesianProductConfigProto.Builder builder = CartesianProductConfigProto.newBuilder(); builder.setIsPartitioned(false).addSources("v0").addSources("v1") - .setMaxParallelism(30).setMinOpsPerWorker(1) - .setNumPartitionsForFairCase(30).setGroupingFraction(0.5f); + .setMaxParallelism(30).setMinOpsPerWorker(1) + .setNumPartitionsForFairCase(30).setGroupingFraction(0.5f); vertexManager.initialize(builder.build()); vertexManager.onVertexStateUpdated(new VertexStateUpdate("v0", VertexState.CONFIGURED)); vertexManager.onVertexStateUpdated(new VertexStateUpdate("v1", VertexState.CONFIGURED)); } + @Test(timeout = 5000) public void testGroupingFraction() throws Exception { setupGroupingFractionTest(); @@ -485,7 +485,7 @@ public void testDisableGrouping() throws Exception { CartesianProductConfigProto.Builder builder = CartesianProductConfigProto.newBuilder(); builder.setIsPartitioned(false).addSources("v0").addSources("v1") - .setMaxParallelism(30).setMinOpsPerWorker(1).setEnableGrouping(false); + .setMaxParallelism(30).setMinOpsPerWorker(1).setEnableGrouping(false); vertexManager.initialize(builder.build()); vertexManager.onVertexStateUpdated(new VertexStateUpdate("v0", VertexState.CONFIGURED)); @@ -513,7 +513,7 @@ public void testParallelismTwoSkewedSource() throws Exception { eq(99), any(), edgePropertiesCaptor.capture()); Map edgeProperties = edgePropertiesCaptor.getValue(); verifyEdgeProperties(edgeProperties.get("v0"), new String[]{"v0", "v1"}, - new int[]{99, 1}, 100); + new int[]{99, 1}, 100); } @Test(timeout = 5000) @@ -523,8 +523,8 @@ public void testParallelismThreeSkewedSource() throws Exception { CartesianProductConfigProto.Builder builder = CartesianProductConfigProto.newBuilder(); builder.setIsPartitioned(false).addSources("v0").addSources("v1").addSources("v2") - .setMaxParallelism(100).setMinOpsPerWorker(10000) - .setNumPartitionsForFairCase(10); + .setMaxParallelism(100).setMinOpsPerWorker(10000) + .setNumPartitionsForFairCase(10); vertexManager.initialize(builder.build()); vertexManager.onVertexStateUpdated(new VertexStateUpdate("v0", VertexState.CONFIGURED)); @@ -541,6 +541,6 @@ public void testParallelismThreeSkewedSource() throws Exception { eq(93), any(), edgePropertiesCaptor.capture()); Map edgeProperties = edgePropertiesCaptor.getValue(); verifyEdgeProperties(edgeProperties.get("v0"), new String[]{"v0", "v1", "v2"}, - new int[]{31, 3, 1}, 100); + new int[]{31, 3, 1}, 100); } } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestGrouper.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestGrouper.java index 08b2efa70b..d17929fd0b 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestGrouper.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/cartesianproduct/TestGrouper.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/TestConfigUtils.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/TestConfigUtils.java index 24f76cc832..5d3caa570d 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/TestConfigUtils.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/TestConfigUtils.java @@ -2,9 +2,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,7 +26,6 @@ import static org.junit.Assert.assertEquals; - public class TestConfigUtils { private static class CustomKey implements WritableComparable, Configurable { @@ -82,4 +81,4 @@ public void getIntermediateInputKeyComparator() { CustomKey customKey = (CustomKey) rawComparator.newKey(); assertEquals(testValue, customKey.getConf().get(testKey)); } -} \ No newline at end of file +} diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/TestInputIdentifiers.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/TestInputIdentifiers.java index 6b82a9d27d..d8df504dc6 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/TestInputIdentifiers.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/TestInputIdentifiers.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,7 +25,7 @@ import org.junit.Test; public class TestInputIdentifiers { - + @Test(timeout = 5000) public void testInputAttemptIdentifier() { Set set = new HashSet(); @@ -33,12 +33,11 @@ public void testInputAttemptIdentifier() { InputAttemptIdentifier i2 = new InputAttemptIdentifier(1, 1, null); InputAttemptIdentifier i3 = new InputAttemptIdentifier(1, 0, null); InputAttemptIdentifier i4 = new InputAttemptIdentifier(0, 1, null); - + Assert.assertTrue(set.add(i1)); Assert.assertFalse(set.add(i1)); Assert.assertFalse(set.add(i2)); Assert.assertTrue(set.add(i3)); Assert.assertTrue(set.add(i4)); } - } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/TestValuesIterator.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/TestValuesIterator.java index eaded181b9..81a1c6b0ac 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/TestValuesIterator.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/TestValuesIterator.java @@ -131,7 +131,7 @@ enum TestWithComparator { * @throws IOException */ public TestValuesIterator(String serializationClassName, Class key, Class val, - TestWithComparator comparator, TestWithComparator correctComparator, boolean testResult) + TestWithComparator comparator, TestWithComparator correctComparator, boolean testResult) throws IOException { this.comparator = getComparator(comparator); this.correctComparator = @@ -223,7 +223,7 @@ private void getNextFromFinishedIterator(ValuesIterator iterator) { try { boolean hasNext = iterator.moveToNext(); fail(); - } catch(IOException e) { + } catch (IOException e) { assertTrue(e.getMessage().contains("Please check if you are invoking moveToNext()")); } } @@ -253,12 +253,11 @@ comparator, new ProgressReporter(), new GenericCounter("readsCounter", "y"), (TezCounter) new GenericCounter("inputValueCounter", "y4")); } - /** * Tests whether data in valuesIterator matches with sorted input data set. - * + *

    * Returns a list of value counts for each key. - * + * * @param valuesIterator * @return List * @throws IOException @@ -274,7 +273,7 @@ private List verifyIteratorData( ListMultimap sortedMap = new ImmutableListMultimap.Builder() .orderKeysBy(this.correctComparator).putAll - (originalData).build(); + (originalData).build(); Set> oriKeySet = Sets.newSet(); oriKeySet.addAll(sortedMap.entries()); @@ -314,7 +313,7 @@ private List verifyIteratorData( assertFalse(valuesIterator.moveToNext()); getNextFromFinishedIterator(valuesIterator); } else { - while(valuesIterator.moveToNext()) { + while (valuesIterator.moveToNext()) { //iterate through all keys } getNextFromFinishedIterator(valuesIterator); @@ -391,13 +390,13 @@ public static Collection getParameters() { //parameters for constructor parameters.add(new Object[] - { null, Text.class, Text.class, TestWithComparator.TEXT, null, true }); + {null, Text.class, Text.class, TestWithComparator.TEXT, null, true}); parameters.add(new Object[] - { null, LongWritable.class, Text.class, TestWithComparator.LONG, null, true }); + {null, LongWritable.class, Text.class, TestWithComparator.LONG, null, true}); parameters.add(new Object[] - { null, IntWritable.class, Text.class, TestWithComparator.INT, null, true }); + {null, IntWritable.class, Text.class, TestWithComparator.INT, null, true}); parameters.add(new Object[] - { null, BytesWritable.class, BytesWritable.class, TestWithComparator.BYTES, null, true }); + {null, BytesWritable.class, BytesWritable.class, TestWithComparator.BYTES, null, true}); parameters.add(new Object[] { TEZ_BYTES_SERIALIZATION, BytesWritable.class, BytesWritable.class, @@ -433,20 +432,20 @@ public static Collection getParameters() { private RawComparator getComparator(TestWithComparator comparator) { switch (comparator) { - case LONG: - return new LongWritable.Comparator(); - case INT: - return new IntWritable.Comparator(); - case BYTES: - return new BytesWritable.Comparator(); - case TEZ_BYTES: - return new TezBytesComparator(); - case TEXT: - return new Text.Comparator(); - case CUSTOM: - return new CustomKey.Comparator(); - default: - return null; + case LONG: + return new LongWritable.Comparator(); + case INT: + return new IntWritable.Comparator(); + case BYTES: + return new BytesWritable.Comparator(); + case TEZ_BYTES: + return new TezBytesComparator(); + case TEXT: + return new Text.Comparator(); + case CUSTOM: + return new CustomKey.Comparator(); + default: + return null; } } @@ -564,7 +563,6 @@ private Map createData() { return map; } - private Writable createData(Class c) { if (c.getName().equalsIgnoreCase(BytesWritable.class.getName())) { return new BytesWritable(new BigInteger(256, rnd).toString().getBytes()); @@ -586,7 +584,8 @@ private Writable createData(Class c) { } private static class ProgressReporter implements Progressable { - @Override public void progress() { + @Override + public void progress() { //no impl } } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/comparator/TestProxyComparator.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/comparator/TestProxyComparator.java index c2f6e2db17..36b483074f 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/comparator/TestProxyComparator.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/comparator/TestProxyComparator.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -33,14 +33,14 @@ public class TestProxyComparator { private static final Logger LOG = LoggerFactory.getLogger(TestProxyComparator.class); final static String[] keys = { - "", - "A", "B", - "AA", "BB", "BA", "CB", - "AAA", "BBBB", "CCCCC", - /* utf-8 comparisons */ - "\u00E6AAAA", "\u00F7", "A\u00F7", "\u00F7AAAAAAAAA", - "\u00F7\u00F7", "\u00F7\u00F7\u00E6\u00E6A", - "\u00F7\u00F7\u00E6\u00E6A" + "", + "A", "B", + "AA", "BB", "BA", "CB", + "AAA", "BBBB", "CCCCC", + /* utf-8 comparisons */ + "\u00E6AAAA", "\u00F7", "A\u00F7", "\u00F7AAAAAAAAA", + "\u00F7\u00F7", "\u00F7\u00F7\u00E6\u00E6A", + "\u00F7\u00F7\u00E6\u00E6A" }; private static final void set(BytesWritable bw, String s) { diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/readers/TestUnorderedKVReader.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/readers/TestUnorderedKVReader.java index 42231e988e..68d1ee6993 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/readers/TestUnorderedKVReader.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/readers/TestUnorderedKVReader.java @@ -121,7 +121,8 @@ public void freeResources(FetchedInput fetchedInput) { ShuffleManager manager = mock(ShuffleManager.class); doAnswer(new Answer() { - @Override public Object answer(InvocationOnMock invocationOnMock) throws Throwable { + @Override + public Object answer(InvocationOnMock invocationOnMock) throws Throwable { return (inputs.isEmpty()) ? null : inputs.remove(); } }).when(manager).getNextInput(); @@ -164,7 +165,7 @@ public void testReadingMultipleTimes() throws Exception { try { boolean next = unorderedKVReader.next(); fail(); - } catch(IOException ioe) { + } catch (IOException ioe) { Assert.assertTrue(ioe.getMessage().contains("For usage, please refer to")); } } @@ -188,5 +189,4 @@ public void testInterruptOnNext() throws IOException, InterruptedException { // Expected exception. Any other should fail the test. } } - } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestFetcher.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestFetcher.java index efba8ea6a7..17e5dccfcd 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestFetcher.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestFetcher.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -151,7 +151,7 @@ public void testLocalFetchModeSetting() throws Exception { @Test(timeout = 3000) public void testSetupLocalDiskFetch() throws Exception { - + CompositeInputAttemptIdentifier[] srcAttempts = { new CompositeInputAttemptIdentifier(0, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_0", 1), new CompositeInputAttemptIdentifier(1, 2, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_1", 1), @@ -171,8 +171,8 @@ public void testSetupLocalDiskFetch() throws Exception { createMockInputContext(), null, conf, true, HOST, PORT, false, true, true); ArrayList inputAttemptIdentifiers = new ArrayList<>(); - for(CompositeInputAttemptIdentifier compositeInputAttemptIdentifier : srcAttempts) { - for(int i=0;i> iterator = fetcher.srcAttemptsRemaining.entrySet().iterator(); int count = 0; - while(iterator.hasNext()) { + while (iterator.hasNext()) { String key = iterator.next().getKey(); Assert.assertTrue(expectedSrcAttempts[count++].toString().compareTo(key) == 0); } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java index 3363cb6627..061dd62aab 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/TestShuffleUtils.java @@ -108,7 +108,6 @@ private OutputContext createTezOutputContext() throws IOException { (conf.get(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID, TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID_DEFAULT)); - doReturn(1).when(outputContext).getTaskVertexIndex(); doReturn(1).when(outputContext).getOutputIndex(); doReturn(0).when(outputContext).getDAGAttemptNumber(); @@ -118,7 +117,6 @@ private OutputContext createTezOutputContext() throws IOException { return outputContext; } - @Before public void setup() throws Exception { conf = new Configuration(); @@ -143,9 +141,9 @@ private Path createIndexFile(int numPartitions, boolean allEmptyPartitions) thro TezSpillRecord spillRecord = new TezSpillRecord(numPartitions); long startOffset = 0; long partLen = 200; //compressed - for(int i=0;i + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -163,7 +163,7 @@ public void testMultipleEvents1() throws IOException { Event dme1 = createDataMovementEvent(0, taskIndex1, createEmptyPartitionByteString(0)); int taskIndex2 = 2; Event dme2 = createDataMovementEvent(0, taskIndex2, null); - + List eventList = new LinkedList(); eventList.add(dme1); eventList.add(dme2); @@ -176,7 +176,6 @@ public void testMultipleEvents1() throws IOException { verify(shuffleManager).addKnownInput(eq(HOST), eq(PORT), eq(expectedIdentifier2), eq(0)); } - private InputContext createInputContext() throws IOException { DataOutputBuffer port_dob = new DataOutputBuffer(); port_dob.writeInt(PORT); @@ -208,7 +207,7 @@ public ExecutorService answer(InvocationOnMock invocation) throws Throwable { private ShuffleManager createShuffleManager(InputContext inputContext) throws IOException { Path outDirBase = new Path(".", "outDir"); - String[] outDirs = new String[] { outDirBase.toString() }; + String[] outDirs = new String[]{outDirBase.toString()}; doReturn(outDirs).when(inputContext).getWorkDirs(); conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, inputContext.getWorkDirs()); @@ -342,7 +341,7 @@ public void testPipelinedShuffleEvents_WithEmptyPartitions() throws IOException } private Event createDataMovementEvent(boolean addSpillDetails, int srcIdx, int targetIdx, - int spillId, boolean isLastSpill, BitSet emptyPartitions, int numPartitions, int attemptNum) + int spillId, boolean isLastSpill, BitSet emptyPartitions, int numPartitions, int attemptNum) throws IOException { DataMovementEventPayloadProto.Builder payloadBuilder = DataMovementEventPayloadProto @@ -368,11 +367,11 @@ private Event createDataMovementEvent(boolean addSpillDetails, int srcIdx, int t } ByteBuffer payload = payloadBuilder.build().toByteString().asReadOnlyByteBuffer(); - return DataMovementEvent.create(srcIdx, targetIdx, attemptNum, payload); + return DataMovementEvent.create(srcIdx, targetIdx, attemptNum, payload); } - + private Event createDataMovementEvent(int srcIndex, int targetIndex, - ByteString emptyPartitionByteString) { + ByteString emptyPartitionByteString) { DataMovementEventPayloadProto.Builder builder = DataMovementEventPayloadProto.newBuilder(); builder.setHost(HOST); builder.setPort(PORT); @@ -392,8 +391,7 @@ private ByteString createEmptyPartitionByteString(int... emptyPartitions) throws } ByteString emptyPartitionsBytesString = TezCommonUtils.compressByteArrayToByteString( TezUtilsInternal - .toByteArray(bitSet)); + .toByteArray(bitSet)); return emptyPartitionsBytesString; } - } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleManager.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleManager.java index ba854b9c14..b6059d3034 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleManager.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleManager.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -75,7 +75,6 @@ import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; - public class TestShuffleManager { private static final String FETCHER_HOST = "localhost"; @@ -100,7 +99,7 @@ public void cleanup() { * partitions, wait for some time and then send DataMovementEvents for the * rest of the partitions. Then do the same thing for the next mapper. * Verify ShuffleManager is able to get all the events. - */ + */ @Test(timeout = 50000) public void testMultiplePartitions() throws Exception { final int numOfMappers = 3; @@ -133,7 +132,6 @@ public void testMultiplePartitions() throws Exception { Thread.sleep(500); - // Send the second batch of DataMovementEvents eventList.clear(); for (int j = 0; j < numOfPartitions - firstPart; j++) { @@ -186,7 +184,7 @@ public ExecutorService answer(InvocationOnMock invocation) throws Throwable { return inputContext; } - @Test(timeout=5000) + @Test(timeout = 5000) public void testUseSharedExecutor() throws Exception { InputContext inputContext = createInputContext(); createShuffleManager(inputContext, 2); @@ -198,7 +196,7 @@ public void testUseSharedExecutor() throws Exception { verify(inputContext).createTezFrameworkExecutorService(anyInt(), anyString()); } - @Test (timeout = 20000) + @Test(timeout = 20000) public void testProgressWithEmptyPendingHosts() throws Exception { InputContext inputContext = createInputContext(); final ShuffleManager shuffleManager = spy(createShuffleManager(inputContext, 1)); @@ -207,7 +205,7 @@ public void testProgressWithEmptyPendingHosts() throws Exception { public void run() { try { shuffleManager.run(); - } catch (Exception e) { + } catch (Exception e) { e.printStackTrace(); } } @@ -218,7 +216,7 @@ public void run() { verify(inputContext, atLeast(3)).notifyProgress(); } - @Test (timeout = 200000) + @Test(timeout = 200000) public void testFetchFailed() throws Exception { InputContext inputContext = createInputContext(); final ShuffleManager shuffleManager = spy(createShuffleManager(inputContext, 1)); @@ -248,7 +246,7 @@ public void run() { List capturedList = captor.getAllValues().get(0); Assert.assertEquals("Size was: " + capturedList.size(), capturedList.size(), 1); - InputReadErrorEvent inputEvent = (InputReadErrorEvent)capturedList.get(0); + InputReadErrorEvent inputEvent = (InputReadErrorEvent) capturedList.get(0); Assert.assertEquals("Number of failures was: " + inputEvent.getNumFailures(), inputEvent.getNumFailures(), 1); @@ -268,19 +266,18 @@ public void run() { capturedList = captor.getAllValues().get(1); Assert.assertEquals("Size was: " + capturedList.size(), capturedList.size(), 1); - inputEvent = (InputReadErrorEvent)capturedList.get(0); + inputEvent = (InputReadErrorEvent) capturedList.get(0); Assert.assertEquals("Number of failures was: " + inputEvent.getNumFailures(), inputEvent.getNumFailures(), 2); - schedulerGetHostThread.interrupt(); } private ShuffleManagerForTest createShuffleManager( InputContext inputContext, int expectedNumOfPhysicalInputs) - throws IOException { + throws IOException { Path outDirBase = new Path(".", "outDir"); - String[] outDirs = new String[] { outDirBase.toString() }; + String[] outDirs = new String[]{outDirBase.toString()}; doReturn(outDirs).when(inputContext).getWorkDirs(); conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, inputContext.getWorkDirs()); @@ -315,9 +312,9 @@ private Event createDataMovementEvent(String host, int srcIndex, int targetIndex private static class ShuffleManagerForTest extends ShuffleManager { public ShuffleManagerForTest(InputContext inputContext, Configuration conf, - int numInputs, int bufferSize, boolean ifileReadAheadEnabled, - int ifileReadAheadLength, CompressionCodec codec, - FetchedInputAllocator inputAllocator) throws IOException { + int numInputs, int bufferSize, boolean ifileReadAheadEnabled, + int ifileReadAheadLength, CompressionCodec codec, + FetchedInputAllocator inputAllocator) throws IOException { super(inputContext, conf, numInputs, bufferSize, ifileReadAheadEnabled, ifileReadAheadLength, codec, inputAllocator); } @@ -331,10 +328,10 @@ Fetcher constructFetcherForHost(InputHost inputHost, Configuration conf) { doAnswer(new Answer() { @Override public FetchResult answer(InvocationOnMock invocation) throws Throwable { - for(InputAttemptIdentifier input : fetcher.getSrcAttempts()) { + for (InputAttemptIdentifier input : fetcher.getSrcAttempts()) { ShuffleManagerForTest.this.fetchSucceeded( fetcher.getHost(), input, new TestFetchedInput(input), 0, 0, - 0); + 0); } return mockFetcherResult; } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestSimpleFetchedInputAllocator.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestSimpleFetchedInputAllocator.java index 01faa5df7a..4a74620bd9 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestSimpleFetchedInputAllocator.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestSimpleFetchedInputAllocator.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -36,20 +36,20 @@ public class TestSimpleFetchedInputAllocator { private static final Logger LOG = LoggerFactory.getLogger(TestSimpleFetchedInputAllocator.class); - + @Test(timeout = 5000) public void testInMemAllocation() throws IOException { File localDirs = new File(System.getProperty("test.build.data", "/tmp"), this.getClass().getName()); Configuration conf = new Configuration(); - + long jvmMax = Runtime.getRuntime().maxMemory(); LOG.info("jvmMax: " + jvmMax); - + float bufferPercent = 0.1f; conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, bufferPercent); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MEMORY_LIMIT_PERCENT, 1.0f); conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, localDirs.getAbsolutePath()); - + long inMemThreshold = (long) (bufferPercent * jvmMax); LOG.info("InMemThreshold: " + inMemThreshold); @@ -60,31 +60,27 @@ public void testInMemAllocation() throws IOException { long requestSize = (long) (0.4f * inMemThreshold); long compressedSize = 1l; LOG.info("RequestSize: " + requestSize); - + FetchedInput fi1 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(1, 1)); assertEquals(FetchedInput.Type.MEMORY, fi1.getType()); - - + FetchedInput fi2 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(2, 1)); assertEquals(FetchedInput.Type.MEMORY, fi2.getType()); - - + // Over limit by this point. Next reserve should give back a DISK allocation FetchedInput fi3 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(3, 1)); assertEquals(FetchedInput.Type.DISK, fi3.getType()); - - + // Freed one memory allocation. Next should be mem again. fi1.abort(); fi1.free(); FetchedInput fi4 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(4, 1)); assertEquals(FetchedInput.Type.MEMORY, fi4.getType()); - + // Freed one disk allocation. Next sould be disk again (no mem freed) fi3.abort(); fi3.free(); FetchedInput fi5 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(4, 1)); assertEquals(FetchedInput.Type.DISK, fi5.getType()); } - } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestFetcher.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestFetcher.java index 068c8f7b02..c4a86024b0 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestFetcher.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestFetcher.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -83,7 +83,6 @@ import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; - public class TestFetcher { public static final String SHUFFLE_INPUT_FILE_PREFIX = "shuffle_input_file_"; @@ -139,7 +138,6 @@ public void testInputsReturnedOnConnectionException() throws Exception { verify(scheduler).putBackKnownMapOutput(mapHost, inputAttemptIdentifier); } - @Test(timeout = 5000) public void testLocalFetchModeSetting1() throws Exception { Configuration conf = new TezConfiguration(); @@ -224,7 +222,6 @@ public void testSetupLocalDiskFetch() throws Exception { connectionErrsCounter, wrongReduceErrsCounter, false, false, true, false, createMockInputContext()); FetcherOrderedGrouped spyFetcher = spy(fetcher); - final List srcAttempts = Arrays.asList( new CompositeInputAttemptIdentifier(0, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_0", 1), new CompositeInputAttemptIdentifier(1, 2, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_1", 1), @@ -234,7 +231,7 @@ public void testSetupLocalDiskFetch() throws Exception { ); final int FIRST_FAILED_ATTEMPT_IDX = 2; final int SECOND_FAILED_ATTEMPT_IDX = 4; - final int[] successfulAttemptsIndexes = { 0, 1, 3 }; + final int[] successfulAttemptsIndexes = {0, 1, 3}; doReturn(srcAttempts).when(scheduler).getMapsForHost(host); @@ -243,8 +240,8 @@ public void testSetupLocalDiskFetch() throws Exception { for (int i = 0; i < srcAttempt.getInputIdentifierCount(); i++) { ShuffleScheduler.PathPartition pathPartition = new ShuffleScheduler.PathPartition(srcAttempt.getPathComponent(), host.getPartitionId() + i); pathToIdentifierMap.put(pathPartition, srcAttempt.expand(i)); - } } + } doAnswer(new Answer() { @Override public InputAttemptIdentifier answer(InvocationOnMock invocation) throws Throwable { @@ -289,7 +286,7 @@ public TezIndexRecord answer(InvocationOnMock invocation) throws Throwable { throw new IOException("failing to simulate failure case"); } // match with params for copySucceeded below. - return new TezIndexRecord(p * 10, (p+1) * 1000, (p+2) * 100); + return new TezIndexRecord(p * 10, (p + 1) * 1000, (p + 2) * 100); } }).when(spyFetcher).getIndexRecord(any(), eq(host.getPartitionId() + i)); } @@ -321,7 +318,7 @@ public TezIndexRecord answer(InvocationOnMock invocation) throws Throwable { verify(scheduler).putBackKnownMapOutput(host, srcAttempts.get(SECOND_FAILED_ATTEMPT_IDX)); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testSetupLocalDiskFetchEmptyPartitions() throws Exception { Configuration conf = new TezConfiguration(); ShuffleScheduler scheduler = mock(ShuffleScheduler.class); @@ -407,7 +404,6 @@ public void testSetupLocalDiskFetchAutoReduce() throws Exception { connectionErrsCounter, wrongReduceErrsCounter, false, false, true, false, createMockInputContext()); FetcherOrderedGrouped spyFetcher = spy(fetcher); - final List srcAttempts = Arrays.asList( new CompositeInputAttemptIdentifier(0, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_0", host.getPartitionCount()), new CompositeInputAttemptIdentifier(1, 2, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_1", host.getPartitionCount()), @@ -417,7 +413,7 @@ public void testSetupLocalDiskFetchAutoReduce() throws Exception { ); final int FIRST_FAILED_ATTEMPT_IDX = 2; final int SECOND_FAILED_ATTEMPT_IDX = 4; - final int[] successfulAttemptsIndexes = { 0, 1, 3 }; + final int[] successfulAttemptsIndexes = {0, 1, 3}; doReturn(srcAttempts).when(scheduler).getMapsForHost(host); final ConcurrentMap pathToIdentifierMap @@ -430,15 +426,15 @@ public void testSetupLocalDiskFetchAutoReduce() throws Exception { } doAnswer(new Answer() { - @Override - public InputAttemptIdentifier answer(InvocationOnMock invocation) throws Throwable { - Object[] args = invocation.getArguments(); - String path = (String) args[0]; - int reduceId = (int) args[1]; - return pathToIdentifierMap.get(new ShuffleScheduler.PathPartition(path, reduceId)); - } - }).when(scheduler) - .getIdentifierForFetchedOutput(any(String.class), any(int.class)); + @Override + public InputAttemptIdentifier answer(InvocationOnMock invocation) throws Throwable { + Object[] args = invocation.getArguments(); + String path = (String) args[0]; + int reduceId = (int) args[1]; + return pathToIdentifierMap.get(new ShuffleScheduler.PathPartition(path, reduceId)); + } + }).when(scheduler) + .getIdentifierForFetchedOutput(any(String.class), any(int.class)); doAnswer(new Answer() { @Override @@ -506,10 +502,10 @@ public TezIndexRecord answer(InvocationOnMock invocation) throws Throwable { eq(InputAttemptFetchFailure.fromLocalFetchFailure(srcAttempts.get(FIRST_FAILED_ATTEMPT_IDX).expand(1))), eq(host), eq(true), eq(false)); verify(scheduler).copyFailed(eq( - InputAttemptFetchFailure.fromLocalFetchFailure(srcAttempts.get(SECOND_FAILED_ATTEMPT_IDX).expand(0))), + InputAttemptFetchFailure.fromLocalFetchFailure(srcAttempts.get(SECOND_FAILED_ATTEMPT_IDX).expand(0))), eq(host), eq(true), eq(false)); verify(scheduler).copyFailed(eq( - InputAttemptFetchFailure.fromLocalFetchFailure(srcAttempts.get(SECOND_FAILED_ATTEMPT_IDX).expand(1))), + InputAttemptFetchFailure.fromLocalFetchFailure(srcAttempts.get(SECOND_FAILED_ATTEMPT_IDX).expand(1))), eq(host), eq(true), eq(false)); verify(spyFetcher).putBackRemainingMapOutputs(host); @@ -520,14 +516,14 @@ public TezIndexRecord answer(InvocationOnMock invocation) throws Throwable { } private void verifyCopySucceeded(ShuffleScheduler scheduler, MapHost host, - List srcAttempts, long p, int j) throws + List srcAttempts, long p, int j) throws IOException { // need to verify filename, offsets, sizes wherever they are used. InputAttemptIdentifier srcAttemptToMatch = srcAttempts.get((int) p).expand(j); String filenameToMatch = SHUFFLE_INPUT_FILE_PREFIX + srcAttemptToMatch.getPathComponent(); ArgumentCaptor captureMapOutput = ArgumentCaptor.forClass(MapOutput.class); - verify(scheduler).copySucceeded(eq(srcAttemptToMatch), eq(host), eq((p+2) * 100), - eq((p+1) * 1000), anyLong(), captureMapOutput.capture(), anyBoolean()); + verify(scheduler).copySucceeded(eq(srcAttemptToMatch), eq(host), eq((p + 2) * 100), + eq((p + 1) * 1000), anyLong(), captureMapOutput.capture(), anyBoolean()); // cannot use the equals of MapOutput as it compares id which is private. so doing it manually MapOutput m = captureMapOutput.getAllValues().get(0); @@ -538,7 +534,7 @@ private void verifyCopySucceeded(ShuffleScheduler scheduler, MapHost host, static class FakeHttpConnection extends HttpConnection { public FakeHttpConnection(URL url, - HttpConnectionParams connParams, String logIdentifier, JobTokenSecretManager jobTokenSecretMgr) + HttpConnectionParams connParams, String logIdentifier, JobTokenSecretManager jobTokenSecretMgr) throws IOException { super(url, connParams, logIdentifier, jobTokenSecretMgr); this.connection = mock(HttpURLConnection.class); @@ -584,7 +580,6 @@ public void testWithRetry() throws Exception { connectionErrsCounter, wrongReduceErrsCounter, false, false, true, false, createMockInputContext()); final FetcherOrderedGrouped fetcher = spy(mockFetcher); - final List srcAttempts = Arrays.asList( new InputAttemptIdentifier(0, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_0"), new InputAttemptIdentifier(1, 2, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_1"), @@ -609,7 +604,8 @@ public MapOutput answer(InvocationOnMock invocation) throws Throwable { //Create read timeout when reading data doAnswer(new Answer() { - @Override public Void answer(InvocationOnMock invocation) throws Throwable { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { doReturn(false).when(fetcher).setupConnection(any(MapHost.class), any(Collection.class)); // Simulate read timeout by throwing proper exception throw new FetcherReadTimeoutException("creating fetcher socket read timeout exception"); @@ -618,29 +614,27 @@ public MapOutput answer(InvocationOnMock invocation) throws Throwable { try { fetcher.copyFromHost(host); - } catch(IOException e) { + } catch (IOException e) { //ignore } //setup connection should be called twice (1 for connect and another for retry) verify(fetcher, times(2)).setupConnection(any(MapHost.class), any(Collection.class)); //since copyMapOutput consistently fails, it should call copyFailed once verify(scheduler, times(1)).copyFailed(any(InputAttemptFetchFailure.class), any(MapHost.class), - anyBoolean(), anyBoolean()); + anyBoolean(), anyBoolean()); verify(fetcher, times(1)).putBackRemainingMapOutputs(any(MapHost.class)); verify(scheduler, times(3)).putBackKnownMapOutput(any(MapHost.class), any(InputAttemptIdentifier.class)); - //Verify by stopping the fetcher abruptly try { fetcher.stopped = false; // flag to indicate fetcher stopped fetcher.copyFromHost(host); verify(fetcher, times(2)).putBackRemainingMapOutputs(any(MapHost.class)); - } catch(IOException e) { + } catch (IOException e) { //ignore } - } @Test @@ -687,37 +681,37 @@ public void testAsyncWithException() throws Exception { @Test(timeout = 5000) public void testInputAttemptIdentifierMap() { InputAttemptIdentifier[] srcAttempts = { - new InputAttemptIdentifier(0, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_0", - false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0), - //duplicate entry - new InputAttemptIdentifier(0, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_0", - false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0), - // pipeline shuffle based identifiers, with multiple attempts - new InputAttemptIdentifier(1, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_1", - false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0), - new InputAttemptIdentifier(1, 2, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_1", - false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0), - new InputAttemptIdentifier(1, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_2", - false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 1), - new InputAttemptIdentifier(1, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_3", - false, InputAttemptIdentifier.SPILL_INFO.FINAL_UPDATE, 2), - new InputAttemptIdentifier(2, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_3", - false, InputAttemptIdentifier.SPILL_INFO.FINAL_MERGE_ENABLED, 0) + new InputAttemptIdentifier(0, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_0", + false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0), + //duplicate entry + new InputAttemptIdentifier(0, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_0", + false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0), + // pipeline shuffle based identifiers, with multiple attempts + new InputAttemptIdentifier(1, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_1", + false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0), + new InputAttemptIdentifier(1, 2, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_1", + false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0), + new InputAttemptIdentifier(1, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_2", + false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 1), + new InputAttemptIdentifier(1, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_3", + false, InputAttemptIdentifier.SPILL_INFO.FINAL_UPDATE, 2), + new InputAttemptIdentifier(2, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_3", + false, InputAttemptIdentifier.SPILL_INFO.FINAL_MERGE_ENABLED, 0) }; InputAttemptIdentifier[] expectedSrcAttempts = { - new InputAttemptIdentifier(0, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_0", - false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0), - // pipeline shuffle based identifiers - new InputAttemptIdentifier(1, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_1", - false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0), - new InputAttemptIdentifier(1, 2, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_1", - false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0), - new InputAttemptIdentifier(1, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_2", - false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 1), - new InputAttemptIdentifier(1, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_3", - false, InputAttemptIdentifier.SPILL_INFO.FINAL_UPDATE, 2), - new InputAttemptIdentifier(2, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_3", - false, InputAttemptIdentifier.SPILL_INFO.FINAL_MERGE_ENABLED, 0) + new InputAttemptIdentifier(0, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_0", + false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0), + // pipeline shuffle based identifiers + new InputAttemptIdentifier(1, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_1", + false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0), + new InputAttemptIdentifier(1, 2, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_1", + false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0), + new InputAttemptIdentifier(1, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_2", + false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 1), + new InputAttemptIdentifier(1, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_3", + false, InputAttemptIdentifier.SPILL_INFO.FINAL_UPDATE, 2), + new InputAttemptIdentifier(2, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_3", + false, InputAttemptIdentifier.SPILL_INFO.FINAL_MERGE_ENABLED, 0) }; Configuration conf = new TezConfiguration(); @@ -733,7 +727,7 @@ public void testInputAttemptIdentifierMap() { Assert.assertEquals(expectedSrcAttempts.length, fetcher.remaining.size()); Iterator> iterator = fetcher.remaining.entrySet().iterator(); int count = 0; - while(iterator.hasNext()) { + while (iterator.hasNext()) { String key = iterator.next().getKey(); Assert.assertTrue(expectedSrcAttempts[count++].toString().compareTo(key) == 0); } @@ -774,7 +768,6 @@ private RawLocalFileSystem getRawFs(Configuration conf) { } } - private InputContext createMockInputContext() { InputContext inputContext = mock(InputContext.class); diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestMergeManager.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestMergeManager.java index dde067beb8..6c4fb764f9 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestMergeManager.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestMergeManager.java @@ -62,7 +62,6 @@ public class TestMergeManager { - private static final Logger LOG = LoggerFactory.getLogger(TestMergeManager.class); private static Configuration defaultConf = new TezConfiguration(); @@ -116,42 +115,42 @@ public void testConfigs() throws IOException { conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 2.4f); MergeManager.getInitialMemoryRequirement(conf, maxTaskMem); Assert.fail("Should have thrown wrong buffer percent configuration exception"); - } catch(IllegalArgumentException ie) { + } catch (IllegalArgumentException ie) { } try { conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, -2.4f); MergeManager.getInitialMemoryRequirement(conf, maxTaskMem); Assert.fail("Should have thrown wrong buffer percent configuration exception"); - } catch(IllegalArgumentException ie) { + } catch (IllegalArgumentException ie) { } try { conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 1.4f); MergeManager.getInitialMemoryRequirement(conf, maxTaskMem); Assert.fail("Should have thrown wrong post merge buffer percent configuration exception"); - } catch(IllegalArgumentException ie) { + } catch (IllegalArgumentException ie) { } try { conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, -1.4f); MergeManager.getInitialMemoryRequirement(conf, maxTaskMem); Assert.fail("Should have thrown wrong post merge buffer percent configuration exception"); - } catch(IllegalArgumentException ie) { + } catch (IllegalArgumentException ie) { } try { conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 1.4f); MergeManager.getInitialMemoryRequirement(conf, maxTaskMem); Assert.fail("Should have thrown wrong shuffle fetch buffer percent configuration exception"); - } catch(IllegalArgumentException ie) { + } catch (IllegalArgumentException ie) { } try { conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, -1.4f); MergeManager.getInitialMemoryRequirement(conf, maxTaskMem); Assert.fail("Should have thrown wrong shuffle fetch buffer percent configuration exception"); - } catch(IllegalArgumentException ie) { + } catch (IllegalArgumentException ie) { } //test post merge mem limit @@ -182,7 +181,7 @@ public void testReservationAccounting() throws IOException { InputContext inputContext = createMockInputContext(UUID.randomUUID().toString()); MergeManager mergeManager = new MergeManager(conf, localFs, null, inputContext, null, null, null, null, - mock(ExceptionReporter.class), 2000000, null, false, -1); + mock(ExceptionReporter.class), 2000000, null, false, -1); mergeManager.configureAndStart(); assertEquals(0, mergeManager.getUsedMemory()); assertEquals(0, mergeManager.getCommitMemory()); @@ -201,7 +200,7 @@ public void testReservationAccounting() throws IOException { assertEquals(0, mergeManager.getCommitMemory()); } - @Test(timeout=20000) + @Test(timeout = 20000) public void testIntermediateMemoryMergeAccounting() throws Exception { Configuration conf = new TezConfiguration(defaultConf); conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, false); @@ -267,7 +266,7 @@ public void testDiskMergeWithCodec() throws Throwable { conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, localDir.toString()); LocalDirAllocator localDirAllocator = - new LocalDirAllocator(TezRuntimeFrameworkConfigs.LOCAL_DIRS); + new LocalDirAllocator(TezRuntimeFrameworkConfigs.LOCAL_DIRS); InputContext inputContext = createMockInputContext(UUID.randomUUID().toString()); // Create a mock compressor. We will check if it is used. @@ -275,8 +274,8 @@ public void testDiskMergeWithCodec() throws Throwable { dummyCodec.setConf(conf); MergeManager mergeManager = - new MergeManager(conf, localFs, localDirAllocator, inputContext, null, null, null, null, - mock(ExceptionReporter.class), 2000, dummyCodec, false, -1); + new MergeManager(conf, localFs, localDirAllocator, inputContext, null, null, null, null, + mock(ExceptionReporter.class), 2000, dummyCodec, false, -1); mergeManager.configureAndStart(); assertEquals(0, mergeManager.getUsedMemory()); @@ -351,10 +350,10 @@ public void testIntermediateMemoryMerge() throws Throwable { * - After 3 segment commits, it would trigger mem-to-mem merge. * - All of them can be merged in memory. */ - InputAttemptIdentifier inputAttemptIdentifier1 = new InputAttemptIdentifier(0,0); - InputAttemptIdentifier inputAttemptIdentifier2 = new InputAttemptIdentifier(1,0); - InputAttemptIdentifier inputAttemptIdentifier3 = new InputAttemptIdentifier(2,0); - InputAttemptIdentifier inputAttemptIdentifier4 = new InputAttemptIdentifier(3,0); + InputAttemptIdentifier inputAttemptIdentifier1 = new InputAttemptIdentifier(0, 0); + InputAttemptIdentifier inputAttemptIdentifier2 = new InputAttemptIdentifier(1, 0); + InputAttemptIdentifier inputAttemptIdentifier3 = new InputAttemptIdentifier(2, 0); + InputAttemptIdentifier inputAttemptIdentifier4 = new InputAttemptIdentifier(3, 0); byte[] data1 = generateDataBySize(conf, 10, inputAttemptIdentifier1); byte[] data2 = generateDataBySize(conf, 20, inputAttemptIdentifier2); byte[] data3 = generateDataBySize(conf, 200, inputAttemptIdentifier3); @@ -375,7 +374,6 @@ public void testIntermediateMemoryMerge() throws Throwable { assertEquals(data1.length + data2.length + data3.length + data4.length, mergeManager.getUsedMemory()); - System.arraycopy(data1, 0, mo1.getMemory(), 0, data1.length); System.arraycopy(data2, 0, mo2.getMemory(), 0, data2.length); System.arraycopy(data3, 0, mo3.getMemory(), 0, data3.length); @@ -395,7 +393,6 @@ public void testIntermediateMemoryMerge() throws Throwable { mergeManager.close(true); - /** * Test #2 * - Have 4 segments where all of them can fit into memory, but one of @@ -633,7 +630,7 @@ private byte[] generateDataBySize(Configuration conf, int rawLen, InputAttemptId IFile.Writer writer = new IFile.Writer(new WritableSerialization(), new WritableSerialization(), fsdos, IntWritable.class, IntWritable.class, null, null, null); int i = 0; - while(true) { + while (true) { writer.append(new IntWritable(i), new IntWritable(i)); i++; if (writer.getRawLength() > rawLen) { @@ -641,8 +638,8 @@ private byte[] generateDataBySize(Configuration conf, int rawLen, InputAttemptId } } writer.close(); - int compressedLength = (int)writer.getCompressedLength(); - int rawLength = (int)writer.getRawLength(); + int compressedLength = (int) writer.getCompressedLength(); + int rawLength = (int) writer.getRawLength(); byte[] data = new byte[rawLength]; ShuffleUtils.shuffleToMemory(data, new ByteArrayInputStream(baos.toByteArray()), rawLength, compressedLength, null, false, 0, LOG, inputAttemptIdentifier); @@ -656,7 +653,7 @@ private byte[] generateDataBySizeAndGetBytes(Configuration conf, int rawLen, IFile.Writer writer = new IFile.Writer(new WritableSerialization(), new WritableSerialization(), fsdos, IntWritable.class, IntWritable.class, null, null, null); int i = 0; - while(true) { + while (true) { writer.append(new IntWritable(i), new IntWritable(i)); i++; if (writer.getRawLength() > rawLen) { @@ -664,11 +661,11 @@ private byte[] generateDataBySizeAndGetBytes(Configuration conf, int rawLen, } } writer.close(); - int compressedLength = (int)writer.getCompressedLength(); - int rawLength = (int)writer.getRawLength(); + int compressedLength = (int) writer.getCompressedLength(); + int rawLength = (int) writer.getRawLength(); byte[] data = new byte[rawLength]; ShuffleUtils.shuffleToMemory(data, new ByteArrayInputStream(baos.toByteArray()), - rawLength, compressedLength, null, false, 0, LOG, inputAttemptIdentifier); + rawLength, compressedLength, null, false, 0, LOG, inputAttemptIdentifier); return baos.toByteArray(); } @@ -682,8 +679,8 @@ private byte[] generateData(Configuration conf, int numEntries, writer.append(new IntWritable(i), new IntWritable(i)); } writer.close(); - int compressedLength = (int)writer.getCompressedLength(); - int rawLength = (int)writer.getRawLength(); + int compressedLength = (int) writer.getCompressedLength(); + int rawLength = (int) writer.getRawLength(); byte[] data = new byte[rawLength]; ShuffleUtils.shuffleToMemory(data, new ByteArrayInputStream(baos.toByteArray()), rawLength, compressedLength, null, false, 0, LOG, inputAttemptIdentifier); @@ -698,12 +695,13 @@ public InterruptingThread(MergeManager.OnDiskMerger mergeThread) { this.mergeThread = mergeThread; } - @Override public void run() { - while(this.mergeThread.tmpDir == null) { - //this is tight loop - } + @Override + public void run() { + while (this.mergeThread.tmpDir == null) { + //this is tight loop + } - this.mergeThread.interrupt(); + this.mergeThread.interrupt(); } } @@ -845,12 +843,10 @@ public void testOnDiskMergerFilenames() throws IOException, InterruptedException assertEquals(m1Prefix, m2Prefix); assertNotEquals(m1Prefix, m3Prefix); assertNotEquals(m2Prefix, m3Prefix); - - verify(inputContext, atLeastOnce()).notifyProgress(); + verify(inputContext, atLeastOnce()).notifyProgress(); } - void testLocalDiskMergeMultipleTasks(final boolean interruptInMiddle) throws IOException, InterruptedException { Configuration conf = new TezConfiguration(defaultConf); @@ -908,7 +904,6 @@ public synchronized void closeOnDiskFile(FileChunk file) { createFile(conf, localFs, new Path(srcDir, InputAttemptIdentifier.PATH_PREFIX + "src2.out"), 2, 3, 6); - // Simulating Task 0 fetches partition 0. (targetIndex = 0,1) // Simulating Task 1 fetches partition 1. (targetIndex = 0,1) @@ -923,7 +918,6 @@ public synchronized void closeOnDiskFile(FileChunk file) { InputAttemptIdentifier t1Identifier1 = new InputAttemptIdentifier(1, 0, src2Info.path.getName()); - MapOutput t0MapOutput0 = getMapOutputForDirectDiskFetch(t0Identifier0, src1Info.path, src1Info.indexedRecords[0], t0mergeManager); @@ -938,7 +932,6 @@ public synchronized void closeOnDiskFile(FileChunk file) { getMapOutputForDirectDiskFetch(t1Identifier1, src2Info.path, src2Info.indexedRecords[1], t1mergeManager); - t0MapOutput0.commit(); t0MapOutput1.commit(); verify(t0mergeManager).closeOnDiskFile(t0MapOutput0.getOutputPath()); @@ -1042,5 +1035,4 @@ private static MapOutput getMapOutputForDirectDiskFetch(InputAttemptIdentifier s return MapOutput.createLocalDiskMapOutput(srcAttemptId, merger, filename, indexRecord.getStartOffset(), indexRecord.getPartLength(), true); } - } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffle.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffle.java index 590affc078..8ce827d47d 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffle.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffle.java @@ -100,12 +100,9 @@ public void testSchedulerTerminatesOnException() throws IOException, Interrupted Throwable t = throwableArgumentCaptor.getValue(); assertTrue(t.getCause().getMessage().contains(exceptionMessage)); - } finally { shuffle.shutdown(); } - - } @Test(timeout = 10000) @@ -135,10 +132,8 @@ public void testKillSelf() throws IOException, InterruptedException { } finally { shuffle.shutdown(); } - } - private InputContext createTezInputContext() throws IOException { ApplicationId applicationId = ApplicationId.newInstance(1, 1); InputContext inputContext = mock(InputContext.class); diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleInputEventHandlerOrderedGrouped.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleInputEventHandlerOrderedGrouped.java index 8da4adcf48..d3f30b7252 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleInputEventHandlerOrderedGrouped.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleInputEventHandlerOrderedGrouped.java @@ -103,35 +103,35 @@ public ExecutorService answer(InvocationOnMock invocation) throws Throwable { } private Event createDataMovementEvent(int srcIndex, int targetIndex, - ByteString emptyPartitionByteString, boolean allPartitionsEmpty) { + ByteString emptyPartitionByteString, boolean allPartitionsEmpty) { return createDataMovementEvent(srcIndex, targetIndex, emptyPartitionByteString, allPartitionsEmpty, false, false, 0); } private Event createDataMovementEvent(int srcIndex, int targetIndex, - ByteString emptyPartitionByteString, boolean allPartitionsEmpty, boolean - finalMergeDisabled, boolean incrementalEvent, int spillId) { + ByteString emptyPartitionByteString, boolean allPartitionsEmpty, boolean + finalMergeDisabled, boolean incrementalEvent, int spillId) { return createDataMovementEvent(srcIndex, targetIndex, emptyPartitionByteString, allPartitionsEmpty, finalMergeDisabled, incrementalEvent, spillId, HOST, PORT); } private Event createDataMovementEvent(int srcIndex, int targetIndex, - ByteString emptyPartitionByteString, boolean allPartitionsEmpty, boolean - finalMergeDisabled, boolean incrementalEvent, int spillId, int attemptNum) { + ByteString emptyPartitionByteString, boolean allPartitionsEmpty, boolean + finalMergeDisabled, boolean incrementalEvent, int spillId, int attemptNum) { return createDataMovementEvent(srcIndex, targetIndex, emptyPartitionByteString, allPartitionsEmpty, finalMergeDisabled, incrementalEvent, spillId, HOST, PORT, attemptNum); } private Event createDataMovementEvent(int srcIndex, int targetIndex, - ByteString emptyPartitionByteString, boolean allPartitionsEmpty, boolean - finalMergeDisabled, boolean incrementalEvent, int spillId, String host, int port) { + ByteString emptyPartitionByteString, boolean allPartitionsEmpty, boolean + finalMergeDisabled, boolean incrementalEvent, int spillId, String host, int port) { return createDataMovementEvent(srcIndex, targetIndex, emptyPartitionByteString, allPartitionsEmpty, finalMergeDisabled, incrementalEvent, spillId, host, port, 0); } private Event createDataMovementEvent(int srcIndex, int targetIndex, - ByteString emptyPartitionByteString, boolean allPartitionsEmpty, boolean - finalMergeDisabled, boolean incrementalEvent, int spillId, String host, int port, int attemptNum) { + ByteString emptyPartitionByteString, boolean allPartitionsEmpty, boolean + finalMergeDisabled, boolean incrementalEvent, int spillId, String host, int port, int attemptNum) { ShuffleUserPayloads.DataMovementEventPayloadProto.Builder builder = ShuffleUserPayloads.DataMovementEventPayloadProto .newBuilder(); @@ -183,7 +183,7 @@ private void setupScheduler(int numInputs) throws Exception { mergeManager = mock(MergeManager.class); } - @Test (timeout = 10000) + @Test(timeout = 10000) public void testPiplinedShuffleEvents() throws IOException, InterruptedException { //test with 2 events per input (2 inputs) int attemptNum = 0; @@ -241,7 +241,7 @@ public void testPiplinedShuffleEvents() throws IOException, InterruptedException assertTrue(scheduler.isDone()); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testPiplinedShuffleEvents_WithOutofOrderAttempts() throws IOException, InterruptedException { //Process attempt #1 first int attemptNum = 1; @@ -275,7 +275,7 @@ public void testPiplinedShuffleEvents_WithOutofOrderAttempts() throws IOExceptio verify(scheduler, times(1)).killSelf(any(), any()); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testPipelinedShuffle_WithObsoleteEvents() throws IOException, InterruptedException { //Process attempt #1 first int attemptNum = 1; @@ -349,7 +349,7 @@ public void testAllPartitionsEmpty() throws IOException { handler.handleEvents(events); InputAttemptIdentifier expectedIdentifier = new InputAttemptIdentifier(targetIdx, 0); verify(scheduler).copySucceeded(eq(expectedIdentifier), any(), eq(0L), - eq(0L), eq(0L), any(), eq(true)); + eq(0L), eq(0L), any(), eq(true)); } @Test(timeout = 5000) diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleScheduler.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleScheduler.java index 9df9aaf69b..c857606e34 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleScheduler.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/TestShuffleScheduler.java @@ -77,7 +77,7 @@ public void cleanup() { sharedExecutor.shutdownNow(); } - @Test (timeout = 10000) + @Test(timeout = 10000) public void testNumParallelScheduledFetchers() throws IOException, InterruptedException { InputContext inputContext = createTezInputContext(); Configuration conf = new TezConfiguration(); @@ -114,7 +114,6 @@ public Void call() throws Exception { // wait for all the copies to be scheduled with timeout scheduler.latch.await(2000, TimeUnit.MILLISECONDS); assertEquals(0, scheduler.latch.getCount()); - } finally { scheduler.close(); if (executorFuture != null) { @@ -124,7 +123,7 @@ public Void call() throws Exception { } } - @Test(timeout=5000) + @Test(timeout = 5000) public void testUseSharedExecutor() throws Exception { InputContext inputContext = createTezInputContext(); Configuration conf = new TezConfiguration(); @@ -190,7 +189,7 @@ public Void call() throws Exception { scheduler.copySucceeded(identifiers[i], mapHosts[i], 20, 25, 100, mapOutput, false); scheduler.freeHost(mapHosts[i]); } - + verify(inputContext, atLeast(numInputs)).notifyProgress(); // Ensure the executor exits, and without an error. @@ -253,7 +252,6 @@ public void _testReducerHealth_1(Configuration conf) throws IOException { new MapHost("host" + (i % totalProducerNodes), 10000, i, 1), false, true); } - InputAttemptIdentifier inputAttemptIdentifier = new InputAttemptIdentifier(200, 0, "attempt_"); @@ -312,7 +310,6 @@ public void testReducerHealth_2() throws IOException, InterruptedException { //120 are successful. so remaining is 200 assertEquals(200, scheduler.remainingMaps.get()); - //200 pending to be downloaded. Download 190. for (int i = 0; i < 190; i++) { InputAttemptIdentifier inputAttemptIdentifier = @@ -391,8 +388,6 @@ public void testReducerHealth_2() throws IOException, InterruptedException { verify(shuffle, atLeast(1)).reportException(any()); } - - @Test(timeout = 60000) /** * Scenario @@ -504,7 +499,7 @@ public void testReducerHealth_4() throws IOException { 10000, i, 1), 100, 200, startTime + (i * 100), mapOutput, false); } - //319 succeeds + //319 succeeds for (int i = 64; i < 319; i++) { InputAttemptIdentifier inputAttemptIdentifier = new InputAttemptIdentifier(i, 0, "attempt_"); @@ -545,7 +540,6 @@ public void testReducerHealth_4() throws IOException { scheduler.copyFailed(InputAttemptFetchFailure.fromAttempt(inputAttemptIdentifier), new MapHost("host" + (319 % totalProducerNodes), 10000, 319, 1), false, true); verify(shuffle, times(1)).reportException(any()); - } @Test(timeout = 60000) @@ -610,7 +604,6 @@ public void testReducerHealth_5() throws IOException { verify(shuffle, times(0)).reportException(any()); } - @Test(timeout = 60000) /** * Scenario @@ -626,14 +619,13 @@ public void testReducerHealth_6() throws IOException { Configuration conf = new TezConfiguration(); conf.setBoolean (TezRuntimeConfiguration - .TEZ_RUNTIME_SHUFFLE_FAILED_CHECK_SINCE_LAST_COMPLETION, true); + .TEZ_RUNTIME_SHUFFLE_FAILED_CHECK_SINCE_LAST_COMPLETION, true); _testReducerHealth_6(conf); conf.setBoolean (TezRuntimeConfiguration .TEZ_RUNTIME_SHUFFLE_FAILED_CHECK_SINCE_LAST_COMPLETION, false); _testReducerHealth_6(conf); - } public void _testReducerHealth_6(Configuration conf) throws IOException { @@ -701,7 +693,6 @@ public void _testReducerHealth_6(Configuration conf) throws IOException { //Do not bail out yet. verify(shuffle, atLeast(0)).reportException(any()); } - } @Test(timeout = 60000) @@ -766,7 +757,7 @@ private ShuffleSchedulerForTest createScheduler(long startTime, int final ShuffleSchedulerForTest scheduler = new ShuffleSchedulerForTest(inputContext, conf, numInputs, shuffle, mergeManager, - mergeManager,startTime, null, false, 0, "srcName"); + mergeManager, startTime, null, false, 0, "srcName"); return scheduler; } @@ -798,14 +789,13 @@ public void testPenalty() throws IOException, InterruptedException { MapHost host = scheduler.getHost(); assertFalse("Host identifier mismatch", (host.getHost() + ":" + host.getPort() + ":" + host.getPartitionId()).equalsIgnoreCase("host0:10000")); - //Refree thread would release it after INITIAL_PENALTY timeout Thread.sleep(ShuffleScheduler.INITIAL_PENALTY + 1000); host = scheduler.getHost(); assertFalse("Host identifier mismatch", (host.getHost() + ":" + host.getPort() + ":" + host.getPartitionId()).equalsIgnoreCase("host0:10000")); } - @Test (timeout = 20000) + @Test(timeout = 20000) public void testProgressDuringGetHostWait() throws IOException, InterruptedException { long startTime = System.currentTimeMillis(); Configuration conf = new TezConfiguration(); @@ -933,8 +923,9 @@ public Void call() throws Exception { try { // Close the scheduler on different thread to trigger interrupt Thread thread = new Thread(new Runnable() { - @Override public void run() { - scheduler.close(); + @Override + public void run() { + scheduler.close(); } }); thread.start(); @@ -946,7 +937,7 @@ public Void call() throws Exception { } } - @Test (timeout = 120000) + @Test(timeout = 120000) public void testPenalties() throws Exception { InputContext inputContext = createTezInputContext(); Configuration conf = new TezConfiguration(); diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestIFile.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestIFile.java index 960aee345a..8f41f01c90 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestIFile.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestIFile.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -127,18 +127,18 @@ public void testWithEmptyIFile() throws IOException { @Test(timeout = 5000) public void testCompressedFlag() throws IOException { - byte[] HEADER = new byte[] { (byte) 'T', (byte) 'I', (byte) 'F' , (byte) 1}; + byte[] HEADER = new byte[]{(byte) 'T', (byte) 'I', (byte) 'F', (byte) 1}; ByteArrayInputStream bin = new ByteArrayInputStream(HEADER); boolean compressed = IFile.Reader.isCompressedFlagEnabled(bin); - assert(compressed == true); + assert (compressed == true); //Negative case: Half cooked header - HEADER = new byte[] { (byte) 'T', (byte) 'I' }; + HEADER = new byte[]{(byte) 'T', (byte) 'I'}; bin = new ByteArrayInputStream(HEADER); try { compressed = IFile.Reader.isCompressedFlagEnabled(bin); fail("Should not have allowed wrong header"); - } catch(Exception e) { + } catch (Exception e) { //correct path. } } @@ -163,8 +163,8 @@ public void testWritingEmptyKeyValues() throws IOException { while (reader.nextRawKey(keyIn)) { reader.nextRawValue(valIn); records++; - assert(keyIn.getLength() == 0); - assert(valIn.getLength() == 0); + assert (keyIn.getLength() == 0); + assert (valIn.getLength() == 0); } assertTrue("Number of records read does not match", (records == 4)); reader.close(); @@ -186,7 +186,6 @@ public void testWithSortedData() throws IOException { testWithDataBuffer(sortedData); } - @Test(timeout = 5000) //test overflow public void testExceedMaxSize() throws IOException { @@ -208,7 +207,7 @@ public void testExceedMaxSize() throws IOException { // Check Key length exceeding MAX_BUFFER_SIZE out = localFs.create(outputPath); writer = new IFile.Writer(new WritableSerialization(), new WritableSerialization(), out, - Text.class, Text.class, null, null, null, false); + Text.class, Text.class, null, null, null, false); writer.append(longString, shortString); writer.close(); @@ -218,7 +217,7 @@ public void testExceedMaxSize() throws IOException { IFile.Reader.MAX_BUFFER_SIZE = 16; reader = new IFile.Reader(localFs, outputPath, - null, null, null, false, 0, -1); + null, null, null, false, 0, -1); try { reader.nextRawKey(keyIn); @@ -231,7 +230,7 @@ public void testExceedMaxSize() throws IOException { // Check Value length exceeding MAX_BUFFER_SIZE out = localFs.create(outputPath); writer = new IFile.Writer(new WritableSerialization(), new WritableSerialization(), out, - Text.class, Text.class, null, null, null, false); + Text.class, Text.class, null, null, null, false); writer.append(shortString, longString); writer.close(); @@ -241,7 +240,7 @@ public void testExceedMaxSize() throws IOException { IFile.Reader.MAX_BUFFER_SIZE = 16; reader = new IFile.Reader(localFs, outputPath, - null, null, null, false, 0, -1); + null, null, null, false, 0, -1); try { reader.nextRawKey(keyIn); @@ -255,7 +254,7 @@ public void testExceedMaxSize() throws IOException { // Check Key length not getting doubled out = localFs.create(outputPath); writer = new IFile.Writer(new WritableSerialization(), new WritableSerialization(), out, - Text.class, Text.class, null, null, null, false); + Text.class, Text.class, null, null, null, false); writer.append(longString, shortString); writer.close(); @@ -265,7 +264,7 @@ public void testExceedMaxSize() throws IOException { IFile.Reader.MAX_BUFFER_SIZE = 32; reader = new IFile.Reader(localFs, outputPath, - null, null, null, false, 0, -1); + null, null, null, false, 0, -1); reader.nextRawKey(keyIn); assertEquals(longString.getLength() + 1, keyIn.getData().length); @@ -274,7 +273,7 @@ public void testExceedMaxSize() throws IOException { // Check Value length not getting doubled out = localFs.create(outputPath); writer = new IFile.Writer(new WritableSerialization(), new WritableSerialization(), out, - Text.class, Text.class, null, null, null, false); + Text.class, Text.class, null, null, null, false); writer.append(shortString, longString); writer.close(); @@ -284,7 +283,7 @@ public void testExceedMaxSize() throws IOException { IFile.Reader.MAX_BUFFER_SIZE = 32; reader = new IFile.Reader(localFs, outputPath, - null, null, null, false, 0, -1); + null, null, null, false, 0, -1); reader.nextRawKey(keyIn); reader.nextRawValue(valIn); @@ -319,7 +318,6 @@ public void testWithRLEMarker() throws IOException { writer.close(); out.close(); - //Test with append(DataInputBuffer key, DataInputBuffer value) byte[] kvbuffer = "key1Value1key1Value2key3Value3".getBytes(); int keyLength = 4; @@ -329,7 +327,7 @@ public void testWithRLEMarker() throws IOException { writer = new IFile.Writer(new WritableSerialization(), new WritableSerialization(), out, Text.class, IntWritable.class, codec, null, null, true); - BoundedByteArrayOutputStream boundedOut = new BoundedByteArrayOutputStream(1024*1024); + BoundedByteArrayOutputStream boundedOut = new BoundedByteArrayOutputStream(1024 * 1024); Writer inMemWriter = new InMemoryWriter(boundedOut, true); DataInputBuffer kin = new DataInputBuffer(); @@ -391,8 +389,8 @@ public void testConcatenatedZlibPadding() byte[] bytes; long compTotal = 0; // Known raw and compressed lengths of input - long raws[] = { 2392, 102314, 42576, 31432, 25090 }; - long compressed[] = { 723, 25396, 10926, 8203, 6665 }; + long raws[] = {2392, 102314, 42576, 31432, 25090}; + long compressed[] = {723, 25396, 10926, 8203, 6665}; CompressionCodecFactory codecFactory = new CompressionCodecFactory(new Configuration()); @@ -495,7 +493,7 @@ public void testAppendValues() throws IOException { Text key = new Text("key"); IntWritable val = new IntWritable(1); - for(int i = 0; i < 5; i++) { + for (int i = 0; i < 5; i++) { data.add(new KVPair(key, val)); values.add(val); } @@ -522,7 +520,7 @@ public void testFileBackedInMemIFileWriter() throws IOException { List values = new ArrayList<>(); Text key = new Text("key"); IntWritable val = new IntWritable(1); - for(int i = 0; i < 5; i++) { + for (int i = 0; i < 5; i++) { data.add(new KVPair(key, val)); values.add(val); } @@ -580,7 +578,7 @@ public void testFileBackedInMemIFileWriter_withSpill() throws IOException { Text key = new Text("key"); IntWritable val = new IntWritable(1); - for(int i = 0; i < 5; i++) { + for (int i = 0; i < 5; i++) { data.add(new KVPair(key, val)); values.add(val); } @@ -638,7 +636,6 @@ public void testEmptyFileBackedInMemIFileWriter() throws IOException { readUsingInMemoryReader(bytes, data); } - @Test(timeout = 5000) //Test appendKeyValues feature public void testAppendKeyValues() throws IOException { @@ -647,7 +644,7 @@ public void testAppendKeyValues() throws IOException { Text key = new Text("key"); IntWritable val = new IntWritable(1); - for(int i = 0; i < 5; i++) { + for (int i = 0; i < 5; i++) { data.add(new KVPair(key, val)); values.add(val); } @@ -876,7 +873,7 @@ private void testWithDataBuffer(List data) throws } private void readAndVerifyData(long rawLength, long compressedLength, - List originalData, CompressionCodec codec) throws + List originalData, CompressionCodec codec) throws IOException { readFileUsingInMemoryReader(rawLength, compressedLength, originalData); readUsingIFileReader(originalData, codec); @@ -891,7 +888,7 @@ private void readAndVerifyData(long rawLength, long compressedLength, * @throws IOException */ private void readFileUsingInMemoryReader(long rawLength, long compressedLength, - List originalData) throws IOException { + List originalData) throws IOException { LOG.info("Read using in memory reader"); FSDataInputStream inStream = localFs.open(outputPath); byte[] bytes = new byte[(int) rawLength]; @@ -918,7 +915,7 @@ private void readUsingInMemoryReader(byte[] bytes, List originalData) * @throws IOException */ private void readUsingIFileReader(List originalData, - CompressionCodec codec) throws IOException { + CompressionCodec codec) throws IOException { LOG.info("Read using IFile reader"); IFile.Reader reader = new IFile.Reader(localFs, outputPath, codec, null, null, false, 0, -1); @@ -970,17 +967,17 @@ private void verifyData(Reader reader, List data) } private Writer writeTestFile(boolean rle, boolean repeatKeys, - List data, CompressionCodec codec) throws IOException { + List data, CompressionCodec codec) throws IOException { FSDataOutputStream out = localFs.create(outputPath); IFile.Writer writer = new IFile.Writer(new WritableSerialization(), new WritableSerialization(), out, Text.class, IntWritable.class, codec, null, null, rle); writeTestFile(writer, repeatKeys, data); out.close(); - return writer; + return writer; } private Writer writeTestFile(IFile.Writer writer, boolean repeatKeys, - List data) throws IOException { + List data) throws IOException { assertNotNull(writer); Text previousKey = null; @@ -1003,7 +1000,7 @@ private Writer writeTestFile(IFile.Writer writer, boolean repeatKeys, } private Writer writeTestFileUsingDataBuffer(boolean rle, boolean repeatKeys, - List data, CompressionCodec codec) throws IOException { + List data, CompressionCodec codec) throws IOException { FSDataOutputStream out = localFs.create(outputPath); IFile.Writer writer = new IFile.Writer(new WritableSerialization(), new WritableSerialization(), out, Text.class, IntWritable.class, codec, null, null, rle); @@ -1013,7 +1010,7 @@ private Writer writeTestFileUsingDataBuffer(boolean rle, boolean repeatKeys, } private Writer writeTestFileUsingDataBuffer(Writer writer, boolean repeatKeys, - List data) throws IOException { + List data) throws IOException { DataInputBuffer previousKey = new DataInputBuffer(); DataInputBuffer key = new DataInputBuffer(); DataInputBuffer value = new DataInputBuffer(); @@ -1037,7 +1034,7 @@ private Writer writeTestFileUsingDataBuffer(Writer writer, boolean repeatKeys, } private void populateData(KVPair kvp, DataInputBuffer key, DataInputBuffer value) - throws IOException { + throws IOException { DataOutputBuffer k = new DataOutputBuffer(); DataOutputBuffer v = new DataOutputBuffer(); kvp.getKey().write(k); diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestPipelinedSorter.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestPipelinedSorter.java index 84ec143808..30516b488c 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestPipelinedSorter.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestPipelinedSorter.java @@ -171,21 +171,20 @@ public void testWithEmptyData() throws IOException { @Test public void testEmptyDataWithPipelinedShuffle() throws IOException { this.numOutputs = 1; - this.initialAvailableMem = 1 *1024 * 1024; + this.initialAvailableMem = 1 * 1024 * 1024; conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, false); conf.setInt(TezRuntimeConfiguration .TEZ_RUNTIME_PIPELINED_SORTER_MIN_BLOCK_SIZE_IN_MB, 1); PipelinedSorter sorter = new PipelinedSorter(this.outputContext, conf, numOutputs, initialAvailableMem); - writeData(sorter, 0, 1<<20); + writeData(sorter, 0, 1 << 20); // final merge is disabled. Final output file would not be populated in this case. assertTrue(sorter.finalOutputFile == null); TezCounter numShuffleChunks = outputContext.getCounters().findCounter(TaskCounter.SHUFFLE_CHUNK_COUNT); // assertTrue(sorter.getNumSpills() == numShuffleChunks.getValue()); conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, true); - } @Test @@ -211,7 +210,7 @@ public void testEmptyPartitionsNoSpillsWithEmptyEvents() throws Exception { public void testEmptyPartitionsHelper(int numKeys, boolean sendEmptyPartitionDetails) throws IOException, InterruptedException { int partitions = 50; this.numOutputs = partitions; - this.initialAvailableMem = 1 *1024 * 1024; + this.initialAvailableMem = 1 * 1024 * 1024; conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED, sendEmptyPartitionDetails); conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, true); conf.setInt(TezRuntimeConfiguration @@ -258,13 +257,13 @@ public void testWithLargeKeyValue() throws IOException { @Test public void testKVExceedsBuffer() throws IOException { // a single block of 1mb, 2KV pair, key 1mb, value 1mb - basicTest(1, 2, (1 << 20), (1 * 1024l * 1024l), 1<<20); + basicTest(1, 2, (1 << 20), (1 * 1024l * 1024l), 1 << 20); } @Test public void testKVExceedsBuffer2() throws IOException { // a list of 4 blocks each 256kb, 2KV pair, key 1mb, value 1mb - basicTest(1, 2, (1 << 20), (1 * 1024l * 1024l), 256<<20); + basicTest(1, 2, (1 << 20), (1 * 1024l * 1024l), 256 << 20); } @Test @@ -277,7 +276,7 @@ public void testExceedsKVWithMultiplePartitions() throws IOException { PipelinedSorter sorter = new PipelinedSorter(this.outputContext, conf, numOutputs, initialAvailableMem); - writeData(sorter, 100, 1<<20); + writeData(sorter, 100, 1 << 20); verifyCounters(sorter, outputContext); verifyOutputPermissions(outputContext.getUniqueIdentifier()); } @@ -285,14 +284,14 @@ public void testExceedsKVWithMultiplePartitions() throws IOException { @Test public void testExceedsKVWithPipelinedShuffle() throws IOException { this.numOutputs = 1; - this.initialAvailableMem = 1 *1024 * 1024; + this.initialAvailableMem = 1 * 1024 * 1024; conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, false); conf.setInt(TezRuntimeConfiguration .TEZ_RUNTIME_PIPELINED_SORTER_MIN_BLOCK_SIZE_IN_MB, 1); PipelinedSorter sorter = new PipelinedSorter(this.outputContext, conf, numOutputs, initialAvailableMem); - writeData(sorter, 5, 1<<20); + writeData(sorter, 5, 1 << 20); // final merge is disabled. Final output file would not be populated in this case. assertTrue(sorter.finalOutputFile == null); @@ -304,7 +303,7 @@ public void testExceedsKVWithPipelinedShuffle() throws IOException { @Test public void test_TEZ_2602_50mb() throws IOException { this.numOutputs = 1; - this.initialAvailableMem = 1 *1024 * 1024; + this.initialAvailableMem = 1 * 1024 * 1024; conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, true); conf.setInt(TezRuntimeConfiguration .TEZ_RUNTIME_PIPELINED_SORTER_MIN_BLOCK_SIZE_IN_MB, 1); @@ -313,7 +312,7 @@ public void test_TEZ_2602_50mb() throws IOException { Text value = new Text("1"); long size = 50 * 1024 * 1024; - while(size > 0) { + while (size > 0) { Text key = RandomTextGenerator.generateSentence(); sorter.write(key, value); size -= key.getLength(); @@ -327,7 +326,7 @@ public void test_TEZ_2602_50mb() throws IOException { //@Test public void testLargeDataWithMixedKV() throws IOException { this.numOutputs = 1; - this.initialAvailableMem = 48 *1024 * 1024; + this.initialAvailableMem = 48 * 1024 * 1024; conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, true); PipelinedSorter sorter = new PipelinedSorter(this.outputContext, conf, numOutputs, initialAvailableMem); @@ -352,7 +351,6 @@ public void testLargeDataWithMixedKV() throws IOException { verifyOutputPermissions(outputContext.getUniqueIdentifier()); } - @Test // first write a KV which dosnt fit into span, this will spill to disk // next write smaller keys, which will update the hint @@ -368,7 +366,7 @@ public void testWithVariableKVLength1() throws IOException { public void testWithVariableKVLength() throws IOException { //2 KVpairs of 2X2mb, 2 KV of 2X7mb int numkeys[] = {2, 2}; - int keylens[] = {2 << 20, 7<<20}; + int keylens[] = {2 << 20, 7 << 20}; basicTest2(1, numkeys, keylens, 64 << 20, 32 << 20); } @@ -379,7 +377,7 @@ public void testWithVariableKVLength() throws IOException { public void testWithVariableKVLength2() throws IOException { // 20 KVpairs of 2X10kb, 10 KV of 2X200kb, 20KV of 2X10kb int numkeys[] = {20, 10, 20}; - int keylens[] = {10<<10, 200<<10, 10<<10}; + int keylens[] = {10 << 10, 200 << 10, 10 << 10}; basicTest2(1, numkeys, keylens, (10 * 1024l * 1024l), 2); } @@ -394,7 +392,7 @@ public void testWithCustomComparator() throws IOException { @Test public void testWithPipelinedShuffle() throws IOException { this.numOutputs = 1; - this.initialAvailableMem = 5 *1024 * 1024; + this.initialAvailableMem = 5 * 1024 * 1024; conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 5); conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, false); conf.setInt(TezRuntimeConfiguration @@ -452,9 +450,9 @@ public void testWithCombiner() throws IOException { this.numOutputs = 5; this.initialAvailableMem = 5 * 1024 * 1024; conf.setInt(TezRuntimeConfiguration - .TEZ_RUNTIME_PIPELINED_SORTER_MIN_BLOCK_SIZE_IN_MB, 3); + .TEZ_RUNTIME_PIPELINED_SORTER_MIN_BLOCK_SIZE_IN_MB, 3); PipelinedSorter sorter = new PipelinedSorter(this.outputContext, conf, numOutputs, - initialAvailableMem); + initialAvailableMem); writeData(sorter, 1, 20); @@ -517,16 +515,16 @@ public void basicTestForBufferUsage() throws IOException { // Check if all buffers are evenly used int avg = (int) sorter.bufferUsage.stream().mapToDouble(d -> d).average().orElse(0.0); - for(int i = 0; i< sorter.bufferUsage.size(); i++) { + for (int i = 0; i < sorter.bufferUsage.size(); i++) { int usage = sorter.bufferUsage.get(i); Assert.assertTrue("Buffer index " + i + " is not used correctly. " - + " usage: " + usage + ", avg: " + avg, usage >= avg); + + " usage: " + usage + ", avg: " + avg, usage >= avg); } conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SORTER_LAZY_ALLOCATE_MEMORY, false); } public void basicTest2(int partitions, int[] numkeys, int[] keysize, - long initialAvailableMem, int blockSize) throws IOException { + long initialAvailableMem, int blockSize) throws IOException { this.numOutputs = partitions; // single output conf.setInt(TezRuntimeConfiguration .TEZ_RUNTIME_PIPELINED_SORTER_MIN_BLOCK_SIZE_IN_MB, 100); @@ -537,7 +535,7 @@ public void basicTest2(int partitions, int[] numkeys, int[] keysize, } private void writeData2(ExternalSorter sorter, - int[] numKeys, int[] keyLen) throws IOException { + int[] numKeys, int[] keyLen) throws IOException { sortedDataMap.clear(); int counter = 0; for (int numkey : numKeys) { @@ -555,7 +553,7 @@ private void writeData2(ExternalSorter sorter, } public void basicTest(int partitions, int numKeys, int keySize, - long initialAvailableMem, int minBlockSize) throws IOException { + long initialAvailableMem, int minBlockSize) throws IOException { this.numOutputs = partitions; // single output conf.setInt(TezRuntimeConfiguration .TEZ_RUNTIME_PIPELINED_SORTER_MIN_BLOCK_SIZE_IN_MB, minBlockSize >> 20); @@ -567,8 +565,8 @@ public void basicTest(int partitions, int numKeys, int keySize, //partition stats; ReportPartitionStats partitionStats = ReportPartitionStats.fromString(conf.get( - TezRuntimeConfiguration.TEZ_RUNTIME_REPORT_PARTITION_STATS, - TezRuntimeConfiguration.TEZ_RUNTIME_REPORT_PARTITION_STATS_DEFAULT)); + TezRuntimeConfiguration.TEZ_RUNTIME_REPORT_PARTITION_STATS, + TezRuntimeConfiguration.TEZ_RUNTIME_REPORT_PARTITION_STATS_DEFAULT)); if (partitionStats.isEnabled()) { assertTrue(sorter.getPartitionStats() != null); } @@ -622,7 +620,6 @@ private void verifyCounters(PipelinedSorter sorter, OutputContext context) { assertTrue(outputBytesWithOverheadCounter.getValue() >= 0); } - @Test //Intentionally not having timeout //Its not possible to allocate > 2 GB in test environment. Carry out basic checks here. @@ -678,7 +675,7 @@ public void memTest() throws IOException { try { sorter = new PipelinedSorter(this.outputContext, conf, numOutputs, (300 << 20)); - } catch(IllegalArgumentException iae ) { + } catch (IllegalArgumentException iae) { assertTrue(iae.getMessage().contains("positive value between 0 and 2047")); } @@ -716,7 +713,7 @@ public void test_without_lazyMemAllocation() throws IOException { //128 MB. Pre-allocate. Request for default block size. Get 1 buffer conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 128); conf.setInt(TezRuntimeConfiguration - .TEZ_RUNTIME_PIPELINED_SORTER_MIN_BLOCK_SIZE_IN_MB, + .TEZ_RUNTIME_PIPELINED_SORTER_MIN_BLOCK_SIZE_IN_MB, TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SORTER_MIN_BLOCK_SIZE_IN_MB_DEFAULT); conf.setBoolean(TezRuntimeConfiguration .TEZ_RUNTIME_PIPELINED_SORTER_LAZY_ALLOCATE_MEMORY, false); @@ -764,7 +761,7 @@ public void test_with_lazyMemAllocation() throws IOException { assertTrue("Expected 1 sort buffers. current len=" + sorter.buffers.size(), sorter.buffers.size() == 1); assertTrue(sorter.buffers.get(0).capacity() == 32 * 1024 * 1024 - 64); - writeData(sorter, 100, 1024*1024, false); //100 1 MB KV. Will spill + writeData(sorter, 100, 1024 * 1024, false); //100 1 MB KV. Will spill //Now it should have created 2 buffers, 32 & 96 MB buffers. assertTrue(sorter.buffers.size() == 2); @@ -783,7 +780,7 @@ public void test_with_lazyMemAllocation() throws IOException { assertTrue(sorter.buffers.size() == 1); assertTrue(sorter.buffers.get(0).capacity() == 32 * 1024 * 1024 - 64); - writeData(sorter, 50, 1024*1024, false); //50 1 MB KV to allocate 2nd buf + writeData(sorter, 50, 1024 * 1024, false); //50 1 MB KV to allocate 2nd buf assertTrue(sorter.buffers.size() == 2); assertTrue(sorter.buffers.get(0).capacity() == 32 * 1024 * 1024 - 64); assertTrue(sorter.buffers.get(1).capacity() == 268 * 1024 * 1024 + 64); @@ -798,7 +795,7 @@ public void test_with_lazyMemAllocation() throws IOException { assertTrue("Expected 1 sort buffers. current len=" + sorter.buffers.size(), sorter.buffers.size() == 1); assertTrue(sorter.buffers.get(0).capacity() == 32 * 1024 * 1024 - 64); - writeData(sorter, 20, 1024*1024, false); //100 1 MB KV. Will spill + writeData(sorter, 20, 1024 * 1024, false); //100 1 MB KV. Will spill //Now it should have created 2 buffers, 32 & 96 MB buffers. assertTrue(sorter.buffers.size() == 2); @@ -848,7 +845,6 @@ public void testLazyAllocateMem() throws IOException { .TEZ_RUNTIME_PIPELINED_SORTER_MIN_BLOCK_SIZE_IN_MB)); assertTrue(iae.getMessage().contains("value between 0 and 2047")); } - } @Test @@ -863,9 +859,9 @@ private void verifyOutputPermissions(String spillId) throws IOException { + "/" + Constants.TEZ_RUNTIME_TASK_OUTPUT_FILENAME_STRING; Path outputPath = dirAllocator.getLocalPathToRead(subpath, conf); Path indexPath = dirAllocator.getLocalPathToRead(subpath + Constants.TEZ_RUNTIME_TASK_OUTPUT_INDEX_SUFFIX_STRING, conf); - Assert.assertEquals("Incorrect output permissions", (short)0640, + Assert.assertEquals("Incorrect output permissions", (short) 0640, localFs.getFileStatus(outputPath).getPermission().toShort()); - Assert.assertEquals("Incorrect index permissions", (short)0640, + Assert.assertEquals("Incorrect index permissions", (short) 0640, localFs.getFileStatus(indexPath).getPermission().toShort()); } @@ -875,7 +871,7 @@ private void writeData(ExternalSorter sorter, int numKeys, int keyLen) throws IO // duplicate some of the keys private void writeSimilarKeys(ExternalSorter sorter, int numKeys, int keyLen, - boolean autoClose) throws IOException { + boolean autoClose) throws IOException { sortedDataMap.clear(); char[] buffer = new char[keyLen]; String keyStr = randomAlphanumeric(buffer); @@ -892,19 +888,21 @@ private void writeSimilarKeys(ExternalSorter sorter, int numKeys, int keyLen, closeSorter(sorter); } } + static private final Random RANDOM = new Random(); int start = ' '; int end = 'z' + 1; int gap = end - start; + private String randomAlphanumeric(char[] buffer) { for (int i = 0; i < buffer.length; ++i) { - buffer[i] = (char)(RANDOM.nextInt(gap) + start); + buffer[i] = (char) (RANDOM.nextInt(gap) + start); } return new String(buffer); } private void writeData(ExternalSorter sorter, int numKeys, int keyLen, - boolean autoClose) throws IOException { + boolean autoClose) throws IOException { char[] buffer = new char[keyLen]; sortedDataMap.clear(); for (int i = 0; i < numKeys; i++) { @@ -955,7 +953,7 @@ private void verifyData(IFile.Reader reader) } private static OutputContext createMockOutputContext(TezCounters counters, ApplicationId appId, - String uniqueId, String auxiliaryService) throws IOException { + String uniqueId, String auxiliaryService) throws IOException { OutputContext outputContext = mock(OutputContext.class); ExecutionContext execContext = new ExecutionContextImpl("localhost"); @@ -979,7 +977,7 @@ private static OutputContext createMockOutputContext(TezCounters counters, Appli doReturn("vertexName").when(outputContext).getTaskVertexName(); doReturn(uniqueId).when(outputContext).getUniqueIdentifier(); Path outDirBase = new Path(workDir, "outDir_" + uniqueId); - String[] outDirs = new String[] { outDirBase.toString() }; + String[] outDirs = new String[]{outDirBase.toString()}; doReturn(outDirs).when(outputContext).getWorkDirs(); return outputContext; } @@ -987,10 +985,10 @@ private static OutputContext createMockOutputContext(TezCounters counters, Appli /** * E.g Hive uses TezBytesComparator which internally makes use of WritableComparator's comparison. * Any length mismatches are handled there. - * + *

    * However, custom comparators can handle this differently and might throw * IndexOutOfBoundsException in case of invalid lengths. - * + *

    * This comparator (similar to comparator in BinInterSedes of pig) would thrown exception when * wrong lengths are mentioned. */ @@ -1003,6 +1001,5 @@ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { return bb1.compareTo(bb2); } - } } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestTezMerger.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestTezMerger.java index c6574d74f4..07610f11b7 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestTezMerger.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestTezMerger.java @@ -220,13 +220,13 @@ public void testWithCustomComparator_WithEmptyStrings() throws Exception { String[][] expectedResult = { //formatting intentionally - { "", DIFF_KEY }, - { "0", DIFF_KEY }, - { "0", SAME_KEY }, - { "0", SAME_KEY }, - { "0", SAME_KEY }, - { "1", DIFF_KEY }, - { "2", DIFF_KEY } + {"", DIFF_KEY}, + {"0", DIFF_KEY}, + {"0", SAME_KEY}, + {"0", SAME_KEY}, + {"0", SAME_KEY}, + {"1", DIFF_KEY}, + {"2", DIFF_KEY} }; verify(records, expectedResult); @@ -267,15 +267,15 @@ public void testWithCustomComparator_No_RLE() throws Exception { String[][] expectedResult = { - { "1", DIFF_KEY }, - { "2", DIFF_KEY }, - { "3", DIFF_KEY }, - { "4", DIFF_KEY }, - { "5", DIFF_KEY }, - { "6", DIFF_KEY }, - { "7", DIFF_KEY }, - { "8", DIFF_KEY }, - { "9", DIFF_KEY } + {"1", DIFF_KEY}, + {"2", DIFF_KEY}, + {"3", DIFF_KEY}, + {"4", DIFF_KEY}, + {"5", DIFF_KEY}, + {"6", DIFF_KEY}, + {"7", DIFF_KEY}, + {"8", DIFF_KEY}, + {"9", DIFF_KEY} }; verify(records, expectedResult); @@ -311,16 +311,15 @@ public void testWithCustomComparator_RLE_acrossFiles() throws Exception { String[][] expectedResult = { //formatting intentionally - { "0", DIFF_KEY }, - { "0", SAME_KEY }, - { "0", SAME_KEY }, - { "1", DIFF_KEY } + {"0", DIFF_KEY}, + {"0", SAME_KEY}, + {"0", SAME_KEY}, + {"1", DIFF_KEY} }; verify(records, expectedResult); pathList.clear(); data.clear(); - } @Test(timeout = 5000) @@ -360,11 +359,11 @@ public void testWithCustomComparator_mixedFiles() throws Exception { String[][] expectedResult = { //formatting intentionally - { "", DIFF_KEY }, - { "0", DIFF_KEY }, - { "0", SAME_KEY }, - { "0", SAME_KEY }, - { "0", SAME_KEY } + {"", DIFF_KEY}, + {"0", DIFF_KEY}, + {"0", SAME_KEY}, + {"0", SAME_KEY}, + {"0", SAME_KEY} }; verify(records, expectedResult); @@ -399,11 +398,11 @@ public void testWithCustomComparator_RLE() throws Exception { String[][] expectedResult = { //formatting intentionally - { "1", DIFF_KEY }, - { "2", DIFF_KEY }, - { "2", SAME_KEY }, - { "2", SAME_KEY }, - { "3", DIFF_KEY } + {"1", DIFF_KEY}, + {"2", DIFF_KEY}, + {"2", SAME_KEY}, + {"2", SAME_KEY}, + {"3", DIFF_KEY} }; verify(records, expectedResult); @@ -444,15 +443,14 @@ public void testWithCustomComparator_RLE2() throws Exception { String[][] expectedResult = { //formatting intentionally - { "0", DIFF_KEY }, - { "0", SAME_KEY }, - { "0", SAME_KEY }, - { "1", DIFF_KEY }, - { "1", SAME_KEY }, - { "1", SAME_KEY }, - { "1", SAME_KEY }, - { "1", SAME_KEY } - + {"0", DIFF_KEY}, + {"0", SAME_KEY}, + {"0", SAME_KEY}, + {"1", DIFF_KEY}, + {"1", SAME_KEY}, + {"1", SAME_KEY}, + {"1", SAME_KEY}, + {"1", SAME_KEY} }; verify(records, expectedResult); @@ -488,9 +486,9 @@ public void testWithCustomComparator() throws Exception { String[][] expectedResult = { //formatting intentionally - { "0", DIFF_KEY }, - { "0", SAME_KEY }, - { "1", DIFF_KEY } + {"0", DIFF_KEY}, + {"0", SAME_KEY}, + {"1", DIFF_KEY} }; verify(records, expectedResult); @@ -523,10 +521,10 @@ public void testWithCustomComparator_RLE3() throws Exception { String[][] expectedResult = { //formatting intentionally - { "0", DIFF_KEY }, - { "0", SAME_KEY }, - { "1", DIFF_KEY }, - { "1", SAME_KEY } }; + {"0", DIFF_KEY}, + {"0", SAME_KEY}, + {"1", DIFF_KEY}, + {"1", SAME_KEY}}; verify(records, expectedResult); pathList.clear(); @@ -579,8 +577,6 @@ private TezRawKeyValueIterator merge(List pathList, RawComparator rc) return records; } - - //Sample comparator to test TEZ-1999 corner case static class CustomComparator extends WritableComparator { @Override @@ -742,8 +738,8 @@ private List createInMemorySegments(int segmentCount, int key } private void populateData(IntWritable intKey, LongWritable longVal, DataInputBuffer key, - DataInputBuffer value) - throws IOException { + DataInputBuffer value) + throws IOException { DataOutputBuffer k = new DataOutputBuffer(); DataOutputBuffer v = new DataOutputBuffer(); intKey.write(k); @@ -828,5 +824,4 @@ private static class Reporter implements Progressable { public void progress() { } } - } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java index a56536dfe8..4fda0aad5c 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java @@ -23,6 +23,7 @@ import org.apache.tez.runtime.library.common.Constants; import org.apache.tez.runtime.library.common.TezRuntimeUtils; import org.junit.Assert; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -137,7 +138,7 @@ public void testSortSpillPercent() throws Exception { try { new DefaultSorter(context, conf, 10, (10 * 1024 * 1024l)); fail(); - } catch(IllegalArgumentException e) { + } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT)); } @@ -145,12 +146,11 @@ public void testSortSpillPercent() throws Exception { try { new DefaultSorter(context, conf, 10, (10 * 1024 * 1024l)); fail(); - } catch(IllegalArgumentException e) { + } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT)); } } - @Test @Ignore /** @@ -227,7 +227,6 @@ public void testSortLimitsWithLargeRecords() throws IOException { } } - @Test(timeout = 5000) public void testSortMBLimits() throws Exception { @@ -240,13 +239,13 @@ public void testSortMBLimits() throws Exception { try { DefaultSorter.computeSortBufferSize(0, ""); fail("Should have thrown error for setting buffer size to 0"); - } catch(RuntimeException re) { + } catch (RuntimeException re) { } try { DefaultSorter.computeSortBufferSize(-100, ""); fail("Should have thrown error for setting buffer size to negative value"); - } catch(RuntimeException re) { + } catch (RuntimeException re) { } } @@ -263,7 +262,7 @@ public void basicTest() throws IOException { ExternalSorter.getInitialMemoryRequirement(conf, context.getTotalMemoryAvailableToTask()), new MemoryUpdateCallbackHandler()); fail(); - } catch(IllegalArgumentException e) { + } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB)); } @@ -283,7 +282,7 @@ public void basicTest() throws IOException { sorterWrapper.close(); assertTrue(sorter.getNumSpills() > 2); verifyCounters(sorter, context); - } catch(IOException ioe) { + } catch (IOException ioe) { fail(ioe.getMessage()); } @@ -292,10 +291,10 @@ public void basicTest() throws IOException { @Test(timeout = 30000) public void testEmptyCaseFileLengths() throws IOException { - testEmptyCaseFileLengthsHelper(50, new String[] {"a", "b"}, new String[] {"1", "2"}); - testEmptyCaseFileLengthsHelper(50, new String[] {"a", "a"}, new String[] {"1", "2"}); - testEmptyCaseFileLengthsHelper(50, new String[] {"aaa", "bbb", "aaa"}, new String[] {"1", "2", "3"}); - testEmptyCaseFileLengthsHelper(1, new String[] {"abcdefghij"}, new String[] {"1234567890"}); + testEmptyCaseFileLengthsHelper(50, new String[]{"a", "b"}, new String[]{"1", "2"}); + testEmptyCaseFileLengthsHelper(50, new String[]{"a", "a"}, new String[]{"1", "2"}); + testEmptyCaseFileLengthsHelper(50, new String[]{"aaa", "bbb", "aaa"}, new String[]{"1", "2", "3"}); + testEmptyCaseFileLengthsHelper(1, new String[]{"abcdefghij"}, new String[]{"1234567890"}); } public void testEmptyCaseFileLengthsHelper(int numPartitions, String[] keys, String[] values) @@ -369,7 +368,7 @@ public void testWithEmptyData() throws IOException { assertTrue(sorter.isClosed()); assertTrue(sorter.getFinalOutputFile().getParent().getName().equalsIgnoreCase(UniqueID)); verifyCounters(sorter, context); - } catch(Exception e) { + } catch (Exception e) { fail(); } } @@ -393,7 +392,7 @@ public void testWithEmptyDataWithFinalMergeDisabled() throws IOException { assertTrue(sorter.getFinalOutputFile().getParent().getName().equalsIgnoreCase(UniqueID + "_0")); verifyCounters(sorter, context); - } catch(Exception e) { + } catch (Exception e) { fail(); } } @@ -524,7 +523,7 @@ public void testWithSingleSpillWithFinalMergeDisabled() throws IOException { ArgumentCaptor eventCaptor = ArgumentCaptor.forClass(List.class); verify(context, times(1)).sendEvents(eventCaptor.capture()); List events = eventCaptor.getValue(); - for(Event event : events) { + for (Event event : events) { if (event instanceof CompositeDataMovementEvent) { CompositeDataMovementEvent cdme = (CompositeDataMovementEvent) event; ShuffleUserPayloads.DataMovementEventPayloadProto shufflePayload = ShuffleUserPayloads @@ -563,7 +562,7 @@ public void testWithMultipleSpillsWithFinalMergeDisabled() throws IOException { verify(context, times(1)).sendEvents(eventCaptor.capture()); List events = eventCaptor.getValue(); int spillIndex = 0; - for(Event event : events) { + for (Event event : events) { if (event instanceof CompositeDataMovementEvent) { CompositeDataMovementEvent cdme = (CompositeDataMovementEvent) event; ShuffleUserPayloads.DataMovementEventPayloadProto shufflePayload = ShuffleUserPayloads @@ -582,9 +581,9 @@ private void verifyOutputPermissions(String spillId) throws IOException { + "/" + Constants.TEZ_RUNTIME_TASK_OUTPUT_FILENAME_STRING; Path outputPath = dirAllocator.getLocalPathToRead(subpath, conf); Path indexPath = dirAllocator.getLocalPathToRead(subpath + Constants.TEZ_RUNTIME_TASK_OUTPUT_INDEX_SUFFIX_STRING, conf); - Assert.assertEquals("Incorrect output permissions", (short)0640, + Assert.assertEquals("Incorrect output permissions", (short) 0640, localFs.getFileStatus(outputPath).getPermission().toShort()); - Assert.assertEquals("Incorrect index permissions", (short)0640, + Assert.assertEquals("Incorrect index permissions", (short) 0640, localFs.getFileStatus(indexPath).getPermission().toShort()); } @@ -627,7 +626,6 @@ private static class SorterWrapper { private final Object[] lastKeys; private final int numPartitions; - public SorterWrapper(OutputContext context, Configuration conf, int numPartitions, long memoryAssigned) throws IOException { sorter = new DefaultSorter(context, conf, numPartitions, memoryAssigned); partitioner = TezRuntimeUtils.instantiatePartitioner(conf); @@ -654,7 +652,7 @@ public int getEmptyPartitionsCount() { return numPartitions - nonEmptyPartitions.cardinality(); } - public void close () throws IOException { + public void close() throws IOException { sorter.flush(); sorter.close(); } @@ -673,7 +671,7 @@ private static Text[] generateData(int numKeys, int keyLen) { } private OutputContext createTezOutputContext() throws IOException { - String[] workingDirs = { workingDir.toString() }; + String[] workingDirs = {workingDir.toString()}; UserPayload payLoad = TezUtils.createUserPayloadFromConf(conf); DataOutputBuffer serviceProviderMetaData = new DataOutputBuffer(); serviceProviderMetaData.writeInt(PORT); @@ -695,7 +693,8 @@ private OutputContext createTezOutputContext() throws IOException { (conf.get(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID, TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID_DEFAULT)); doAnswer(new Answer() { - @Override public Object answer(InvocationOnMock invocation) throws Throwable { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { long requestedSize = (Long) invocation.getArguments()[0]; MemoryUpdateCallbackHandler callback = (MemoryUpdateCallbackHandler) invocation .getArguments()[1]; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java index dc7357c9cc..1d0c49e840 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/writers/TestUnorderedPartitionedKVWriter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -127,7 +127,7 @@ public class TestUnorderedPartitionedKVWriter { private Configuration defaultConf = new Configuration(); public TestUnorderedPartitionedKVWriter(boolean shouldCompress, - ReportPartitionStats reportPartitionStats) { + ReportPartitionStats reportPartitionStats) { this.shouldCompress = shouldCompress; this.reportPartitionStats = reportPartitionStats; } @@ -135,17 +135,17 @@ public TestUnorderedPartitionedKVWriter(boolean shouldCompress, @SuppressWarnings("deprecation") @Parameterized.Parameters(name = "test[{0}, {1}]") public static Collection data() { - Object[][] data = new Object[][] { - { false, ReportPartitionStats.DISABLED }, - { false, ReportPartitionStats.ENABLED }, - { false, ReportPartitionStats.NONE }, - { false, ReportPartitionStats.MEMORY_OPTIMIZED }, - { false, ReportPartitionStats.PRECISE }, - { true, ReportPartitionStats.DISABLED }, - { true, ReportPartitionStats.ENABLED }, - { true, ReportPartitionStats.NONE }, - { true, ReportPartitionStats.MEMORY_OPTIMIZED }, - { true, ReportPartitionStats.PRECISE }}; + Object[][] data = new Object[][]{ + {false, ReportPartitionStats.DISABLED}, + {false, ReportPartitionStats.ENABLED}, + {false, ReportPartitionStats.NONE}, + {false, ReportPartitionStats.MEMORY_OPTIMIZED}, + {false, ReportPartitionStats.PRECISE}, + {true, ReportPartitionStats.DISABLED}, + {true, ReportPartitionStats.ENABLED}, + {true, ReportPartitionStats.NONE}, + {true, ReportPartitionStats.MEMORY_OPTIMIZED}, + {true, ReportPartitionStats.PRECISE}}; return Arrays.asList(data); } @@ -353,8 +353,8 @@ public void testTextMixedRecordsWithoutFinalMerge() throws IOException, Interrup } public void textTest(int numRegularRecords, int numPartitions, long availableMemory, - int numLargeKeys, int numLargevalues, int numLargeKvPairs, - boolean pipeliningEnabled, boolean isFinalMergeEnabled) throws IOException, + int numLargeKeys, int numLargevalues, int numLargeKvPairs, + boolean pipeliningEnabled, boolean isFinalMergeEnabled) throws IOException, InterruptedException { Partitioner partitioner = new HashPartitioner(); ApplicationId appId = ApplicationId.newInstance(10000000, 1); @@ -381,7 +381,7 @@ public void textTest(int numRegularRecords, int numPartitions, long availableMem Map> expectedValues = new HashMap>(); for (int i = 0; i < numPartitions; i++) { - expectedValues.put(i, LinkedListMultimap. create()); + expectedValues.put(i, LinkedListMultimap.create()); } UnorderedPartitionedKVWriter kvWriter = new UnorderedPartitionedKVWriterForTest(outputContext, @@ -465,8 +465,8 @@ public void textTest(int numRegularRecords, int numPartitions, long availableMem } } VertexManagerEventPayloadProto vmEventPayload = - VertexManagerEventPayloadProto.parseFrom( - ByteString.copyFrom(vmEvent.getUserPayload().asReadOnlyBuffer())); + VertexManagerEventPayloadProto.parseFrom( + ByteString.copyFrom(vmEvent.getUserPayload().asReadOnlyBuffer())); assertEquals(numRecordsWritten, vmEventPayload.getNumRecord()); } @@ -572,7 +572,7 @@ private int[] getPartitionStats(VertexManagerEvent vme) throws IOException { List sizeInMBList = payload.getDetailedPartitionStats().getSizeInMbList(); int[] stats = new int[sizeInMBList.size()]; - for (int i=0; i skippedPartitions, boolean shouldCompress) throws IOException, InterruptedException { @@ -716,9 +715,9 @@ private void baseTestWithPipelinedTransfer(int numRecords, int numPartitions, Se verify(outputContext, atLeast(numExpectedSpills)).sendEvents(eventCaptor.capture()); int numOfCapturedEvents = eventCaptor.getAllValues().size(); lastEvents = eventCaptor.getAllValues().get(numOfCapturedEvents - 1); - VertexManagerEvent VMEvent = (VertexManagerEvent)lastEvents.get(0); + VertexManagerEvent VMEvent = (VertexManagerEvent) lastEvents.get(0); - for (int i=0; i events = eventCaptor.getAllValues().get(i); if (i < numOfCapturedEvents - 1) { assertTrue(events.size() == 1); @@ -788,7 +787,7 @@ private void baseTestWithPipelinedTransfer(int numRecords, int numPartitions, Se int index = lastEvents.size() - 1; assertTrue(lastEvents.get(index) instanceof CompositeDataMovementEvent); CompositeDataMovementEvent cdme = - (CompositeDataMovementEvent)lastEvents.get(index); + (CompositeDataMovementEvent) lastEvents.get(index); assertEquals(0, cdme.getSourceIndexStart()); assertEquals(numOutputs, cdme.getCount()); DataMovementEventPayloadProto eventProto = @@ -829,7 +828,7 @@ private void checkPermissions(Path outputFile, Path indexFile) throws IOExceptio } private void verifyEmptyPartitions(DataMovementEventPayloadProto eventProto, - int numRecordsWritten, int numPartitions, Set skippedPartitions) + int numRecordsWritten, int numPartitions, Set skippedPartitions) throws IOException { if (eventProto.hasEmptyPartitions()) { byte[] emptyPartitions = TezCommonUtils.decompressByteStringToByteArray( @@ -898,7 +897,7 @@ public void testSkippedPartitions_WithFinalMergeDisabled() throws IOException, I @SuppressWarnings("unchecked") private void baseTestWithFinalMergeDisabled(int numRecords, int numPartitions, - Set skippedPartitions, boolean shouldCompress) throws IOException, InterruptedException { + Set skippedPartitions, boolean shouldCompress) throws IOException, InterruptedException { PartitionerForTest partitioner = new PartitionerForTest(); ApplicationId appId = ApplicationId.newInstance(10000000, 1); @@ -964,8 +963,8 @@ private void baseTestWithFinalMergeDisabled(int numRecords, int numPartitions, assertEquals((spills + 1), lastEvents.size()); //spills + VMEvent verify(outputContext, atMost(0)).sendEvents(eventCaptor.capture()); - for (int i=0; i 0); if (!shouldCompress) { assertTrue("fileOutputBytes=" + fileOutputBytes + ", outputRecordBytes=" - +outputRecordBytesCounter.getValue(), + + outputRecordBytesCounter.getValue(), fileOutputBytes > outputRecordBytesCounter.getValue()); } } else { @@ -1037,7 +1036,7 @@ private void baseTestWithFinalMergeDisabled(int numRecords, int numPartitions, int index = lastEvents.size() - 1; assertTrue(lastEvents.get(index) instanceof CompositeDataMovementEvent); CompositeDataMovementEvent cdme = - (CompositeDataMovementEvent)lastEvents.get(index); + (CompositeDataMovementEvent) lastEvents.get(index); assertEquals(0, cdme.getSourceIndexStart()); assertEquals(numOutputs, cdme.getCount()); DataMovementEventPayloadProto eventProto = @@ -1053,11 +1052,11 @@ private void baseTestWithFinalMergeDisabled(int numRecords, int numPartitions, // Verify if all path components have spillIds when final merge is disabled Pattern mergePathComponentPattern = Pattern.compile("(.*)(_\\d+)"); - for(Event event : lastEvents) { + for (Event event : lastEvents) { if (!(event instanceof CompositeDataMovementEvent)) { continue; } - cdme = (CompositeDataMovementEvent)event; + cdme = (CompositeDataMovementEvent) event; eventProto = DataMovementEventPayloadProto.parseFrom(ByteString.copyFrom(cdme.getUserPayload())); assertEquals(false, eventProto.getPipelined()); @@ -1085,7 +1084,6 @@ private void baseTestWithFinalMergeDisabled(int numRecords, int numPartitions, } } - verify(outputContext, atLeast(1)).notifyProgress(); // Verify if all spill files are available. @@ -1103,16 +1101,16 @@ private void baseTestWithFinalMergeDisabled(int numRecords, int numPartitions, } private void baseTest(int numRecords, int numPartitions, Set skippedPartitions, - boolean shouldCompress, int maxSingleBufferSizeBytes, int bufferMergePercent) + boolean shouldCompress, int maxSingleBufferSizeBytes, int bufferMergePercent) throws IOException, InterruptedException { baseTest(numRecords, numPartitions, skippedPartitions, shouldCompress, maxSingleBufferSizeBytes, bufferMergePercent, 2048, false); } private void baseTest(int numRecords, int numPartitions, Set skippedPartitions, - boolean shouldCompress, int maxSingleBufferSizeBytes, int bufferMergePercent, int - availableMemory, boolean dataViaEventEnabled) - throws IOException, InterruptedException { + boolean shouldCompress, int maxSingleBufferSizeBytes, int bufferMergePercent, int + availableMemory, boolean dataViaEventEnabled) + throws IOException, InterruptedException { PartitionerForTest partitioner = new PartitionerForTest(); ApplicationId appId = ApplicationId.newInstance(10000000, 1); TezCounters counters = new TezCounters(); @@ -1142,7 +1140,7 @@ private void baseTest(int numRecords, int numPartitions, Set skippedPar Map> expectedValues = new HashMap>(); for (int i = 0; i < numOutputs; i++) { - expectedValues.put(i, LinkedListMultimap. create()); + expectedValues.put(i, LinkedListMultimap.create()); } UnorderedPartitionedKVWriter kvWriter = new UnorderedPartitionedKVWriterForTest(outputContext, @@ -1300,24 +1298,24 @@ private void baseTest(int numRecords, int numPartitions, Set skippedPar return; } - boolean isInMem= eventProto.getData().hasData(); + boolean isInMem = eventProto.getData().hasData(); assertTrue(localFs.exists(outputFilePath)); assertEquals("Incorrect output permissions (user)", FsAction.READ_WRITE, - localFs.getFileStatus(outputFilePath).getPermission().getUserAction()); + localFs.getFileStatus(outputFilePath).getPermission().getUserAction()); assertEquals("Incorrect output permissions (group)", FsAction.READ, - localFs.getFileStatus(outputFilePath).getPermission().getGroupAction()); - if( !isInMem ) { + localFs.getFileStatus(outputFilePath).getPermission().getGroupAction()); + if (!isInMem) { assertTrue(localFs.exists(spillFilePath)); assertEquals("Incorrect index permissions (user)", FsAction.READ_WRITE, - localFs.getFileStatus(spillFilePath).getPermission().getUserAction()); + localFs.getFileStatus(spillFilePath).getPermission().getUserAction()); assertEquals("Incorrect index permissions (group)", FsAction.READ, - localFs.getFileStatus(spillFilePath).getPermission().getGroupAction()); + localFs.getFileStatus(spillFilePath).getPermission().getGroupAction()); // verify no intermediate spill files have been left around synchronized (kvWriter.spillInfoList) { for (SpillInfo spill : kvWriter.spillInfoList) { assertFalse("lingering intermediate spill file " + spill.outPath, - localFs.exists(spill.outPath)); + localFs.exists(spill.outPath)); } } } @@ -1335,7 +1333,7 @@ private void baseTest(int numRecords, int numPartitions, Set skippedPar int dataLoadSize = eventProto.getData().getData().size(); inStream = new ByteArrayInputStream(eventProto.getData().getData().toByteArray()); reader = new IFile.Reader(inStream, dataLoadSize, codec, null, - null, false, 0, -1); + null, false, 0, -1); } else { TezSpillRecord spillRecord = new TezSpillRecord(spillFilePath, conf); TezIndexRecord indexRecord = spillRecord.getIndex(i); @@ -1348,7 +1346,7 @@ private void baseTest(int numRecords, int numPartitions, Set skippedPar tmpStream.seek(indexRecord.getStartOffset()); inStream = tmpStream; reader = new IFile.Reader(tmpStream, indexRecord.getPartLength(), codec, null, - null, false, 0, -1); + null, false, 0, -1); } while (reader.nextRawKey(keyBuffer)) { @@ -1379,7 +1377,7 @@ private static String createRandomString(int size) { } private OutputContext createMockOutputContext(TezCounters counters, ApplicationId appId, - String uniqueId, String auxiliaryService) { + String uniqueId, String auxiliaryService) { OutputContext outputContext = mock(OutputContext.class); doReturn(counters).when(outputContext).getCounters(); doReturn(appId).when(outputContext).getApplicationId(); @@ -1405,22 +1403,22 @@ public ByteBuffer answer(InvocationOnMock invocation) throws Throwable { }).when(outputContext).getServiceProviderMetaData(eq(auxiliaryService)); Path outDirBase = new Path(TEST_ROOT_DIR, "outDir_" + uniqueId); - String[] outDirs = new String[] { outDirBase.toString() }; + String[] outDirs = new String[]{outDirBase.toString()}; doReturn(outDirs).when(outputContext).getWorkDirs(); return outputContext; } private Configuration createConfiguration(OutputContext outputContext, - Class keyClass, Class valClass, - boolean shouldCompress, int maxSingleBufferSizeBytes) { + Class keyClass, Class valClass, + boolean shouldCompress, int maxSingleBufferSizeBytes) { return createConfiguration(outputContext, keyClass, valClass, shouldCompress, maxSingleBufferSizeBytes, PartitionerForTest.class); } private Configuration createConfiguration(OutputContext outputContext, - Class keyClass, Class valClass, - boolean shouldCompress, int maxSingleBufferSizeBytes, - Class partitionerClass) { + Class keyClass, Class valClass, + boolean shouldCompress, int maxSingleBufferSizeBytes, + Class partitionerClass) { Configuration conf = new Configuration(false); conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077"); conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, outputContext.getWorkDirs()); @@ -1456,7 +1454,7 @@ public int getPartition(Object key, Object value, int numPartitions) { private static class UnorderedPartitionedKVWriterForTest extends UnorderedPartitionedKVWriter { public UnorderedPartitionedKVWriterForTest(OutputContext outputContext, Configuration conf, - int numOutputs, long availableMemoryBytes) throws IOException { + int numOutputs, long availableMemoryBytes) throws IOException { super(outputContext, conf, numOutputs, availableMemoryBytes); } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedGroupedMergedKVInputConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedGroupedMergedKVInputConfig.java index 70dbe143c1..04083cdca8 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedGroupedMergedKVInputConfig.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedGroupedMergedKVInputConfig.java @@ -99,7 +99,6 @@ public void testSetters() { OrderedGroupedKVInputConfig configuration = builder.build(); - OrderedGroupedKVInputConfig rebuilt = new OrderedGroupedKVInputConfig(); rebuilt.fromUserPayload(configuration.toUserPayload()); diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfig.java index 9567770815..54a2a7bd4b 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfig.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfig.java @@ -66,7 +66,7 @@ public void testNullParams() { } } - @Test (timeout=2000) + @Test(timeout = 2000) public void testDefaultConfigsUsed() { OrderedPartitionedKVEdgeConfig.Builder builder = OrderedPartitionedKVEdgeConfig .newBuilder("KEY", "VALUE", "PARTITIONER"); @@ -92,7 +92,7 @@ public void testDefaultConfigsUsed() { inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "")); } - @Test (timeout=2000) + @Test(timeout = 2000) public void testSpecificIOConfs() { // Ensures that Output and Input confs are not mixed. OrderedPartitionedKVEdgeConfig.Builder builder = OrderedPartitionedKVEdgeConfig @@ -115,7 +115,7 @@ public void testSpecificIOConfs() { inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "DEFAULT")); } - @Test (timeout=2000) + @Test(timeout = 2000) public void tetCommonConf() { Configuration fromConf = new Configuration(false); @@ -183,11 +183,10 @@ public void tetCommonConf() { ReportPartitionStats partitionStats = ReportPartitionStats.fromString(outputConf.get( - TezRuntimeConfiguration.TEZ_RUNTIME_REPORT_PARTITION_STATS, - TezRuntimeConfiguration.TEZ_RUNTIME_REPORT_PARTITION_STATS_DEFAULT)); + TezRuntimeConfiguration.TEZ_RUNTIME_REPORT_PARTITION_STATS, + TezRuntimeConfiguration.TEZ_RUNTIME_REPORT_PARTITION_STATS_DEFAULT)); assertEquals(true, partitionStats.isEnabled()); - assertEquals(3, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, 0)); assertEquals(1111, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, 0)); assertEquals(2222, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE, 0)); @@ -207,7 +206,7 @@ public void tetCommonConf() { assertEquals("unfiltered1", inputConf.get("test.conf.unfiltered.1")); } - @Test (timeout=2000) + @Test(timeout = 2000) public void testSetters() { Map comparatorConf = Maps.newHashMap(); comparatorConf.put("comparator.test.key", "comparatorValue"); @@ -248,7 +247,6 @@ public void testSetters() { assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT)); assertNull(outputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT)); - assertEquals("KEY_COMPARATOR", inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_COMPARATOR_CLASS)); assertEquals("comparatorValue", inputConf.get("comparator.test.key")); assertEquals("KEY", inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, "")); @@ -269,10 +267,9 @@ public void testSetters() { assertEquals(0.44f, inputConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.00f), 0.001f); assertNull(inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB)); - } - @Test (timeout=2000) + @Test(timeout = 2000) public void testSerialization() { OrderedPartitionedKVEdgeConfig.Builder builder = OrderedPartitionedKVEdgeConfig .newBuilder("KEY", "VALUE", "PARTITIONER") @@ -307,7 +304,6 @@ public void testSerialization() { assertTrue(outputConf.get(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY).trim().startsWith ("serClass2,serClass1")); - //verify comparator and serialization class assertEquals("SomeComparator1", inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_COMPARATOR_CLASS)); assertTrue(inputConf.get(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY).trim().startsWith @@ -329,7 +325,7 @@ private void checkHistoryText(String historyText) { TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT)); } - @Test (timeout=2000) + @Test(timeout = 2000) public void testHistoryText() { OrderedPartitionedKVEdgeConfig.Builder builder = OrderedPartitionedKVEdgeConfig.newBuilder("KEY", "VALUE", "PARTITIONER"); @@ -351,6 +347,4 @@ public void testHistoryText() { checkHistoryText(edgeProperty.getEdgeDestination().getHistoryText()); checkHistoryText(edgeProperty.getEdgeSource().getHistoryText()); } - - } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVEdgeConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVEdgeConfig.java index 3c97020f48..a90df8489a 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVEdgeConfig.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVEdgeConfig.java @@ -39,7 +39,7 @@ public class TestUnorderedKVEdgeConfig { - @Test (timeout=2000) + @Test(timeout = 2000) public void testNullParams() { try { UnorderedKVEdgeConfig.newBuilder(null, "VALUE"); @@ -56,7 +56,7 @@ public void testNullParams() { } } - @Test (timeout=2000) + @Test(timeout = 2000) public void testDefaultConfigsUsed() { UnorderedKVEdgeConfig.Builder builder = UnorderedKVEdgeConfig.newBuilder("KEY", "VALUE"); @@ -64,7 +64,6 @@ public void testDefaultConfigsUsed() { UnorderedKVEdgeConfig configuration = builder.build(); - UnorderedKVOutputConfig rebuiltOutput = new UnorderedKVOutputConfig(); rebuiltOutput.fromUserPayload(configuration.getOutputPayload()); @@ -89,7 +88,7 @@ public void testDefaultConfigsUsed() { ("SerClass2,SerClass1")); } - @Test (timeout=2000) + @Test(timeout = 2000) public void testSpecificIOConfs() { // Ensures that Output and Input confs are not mixed. UnorderedKVEdgeConfig.Builder builder = @@ -113,7 +112,7 @@ public void testSpecificIOConfs() { inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "DEFAULT")); } - @Test (timeout=2000) + @Test(timeout = 2000) public void tetCommonConf() { Configuration fromConf = new Configuration(false); @@ -186,7 +185,7 @@ private void checkHistoryText(String historyText) { TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT)); } - @Test (timeout=2000) + @Test(timeout = 2000) public void testHistoryText() { UnorderedKVEdgeConfig.Builder builder = UnorderedKVEdgeConfig.newBuilder("KEY", "VALUE"); Configuration fromConf = new Configuration(false); @@ -211,8 +210,5 @@ public void testHistoryText() { EdgeProperty edgeProperty = builder.build().createDefaultCustomEdgeProperty(descriptor); checkHistoryText(edgeProperty.getEdgeDestination().getHistoryText()); checkHistoryText(edgeProperty.getEdgeSource().getHistoryText()); - } - - } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfig.java index 684f773c27..f35ce1a5a5 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfig.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfig.java @@ -39,7 +39,7 @@ public class TestUnorderedPartitionedKVEdgeConfig { - @Test (timeout=2000) + @Test(timeout = 2000) public void testNullParams() { try { UnorderedPartitionedKVEdgeConfig.newBuilder(null, "VALUE", "PARTITIONER"); @@ -63,7 +63,7 @@ public void testNullParams() { } } - @Test (timeout=2000) + @Test(timeout = 2000) public void testDefaultConfigsUsed() { UnorderedPartitionedKVEdgeConfig.Builder builder = UnorderedPartitionedKVEdgeConfig.newBuilder("KEY", "VALUE", "PARTITIONER"); @@ -96,7 +96,7 @@ public void testDefaultConfigsUsed() { ("SerClass2,SerClass1")); } - @Test (timeout=2000) + @Test(timeout = 2000) public void testSpecificIOConfs() { // Ensures that Output and Input confs are not mixed. UnorderedPartitionedKVEdgeConfig.Builder builder = @@ -120,7 +120,7 @@ public void testSpecificIOConfs() { inputConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC, "DEFAULT")); } - @Test (timeout=2000) + @Test(timeout = 2000) public void tetCommonConf() { Configuration fromConf = new Configuration(false); @@ -201,7 +201,7 @@ private void checkHistoryText(String historyText) { TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT)); } - @Test (timeout=2000) + @Test(timeout = 2000) public void testHistoryText() { UnorderedPartitionedKVEdgeConfig.Builder builder = UnorderedPartitionedKVEdgeConfig.newBuilder("KEY", "VALUE", "PARTITIONER"); @@ -223,7 +223,5 @@ public void testHistoryText() { EdgeProperty edgeProperty = builder.build().createDefaultCustomEdgeProperty(descriptor); checkHistoryText(edgeProperty.getEdgeDestination().getHistoryText()); checkHistoryText(edgeProperty.getEdgeSource().getHistoryText()); - } - } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVOutputConfig.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVOutputConfig.java index bff2868020..1328ff418d 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVOutputConfig.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVOutputConfig.java @@ -77,7 +77,7 @@ public void testSetters() { fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1"); UnorderedPartitionedKVOutputConfig.Builder builder = UnorderedPartitionedKVOutputConfig.newBuilder("KEY", "VALUE", "PARTITIONER", - null) + null) .setCompression(true, "CustomCodec", null) .setAvailableBufferSize(1111) .setAdditionalConfiguration("fs.shouldExist", "fs") @@ -87,7 +87,7 @@ public void testSetters() { .setAdditionalConfiguration(TezRuntimeConfiguration .TEZ_RUNTIME_TRANSFER_DATA_VIA_EVENTS_MAX_SIZE, "5120") .setAdditionalConfiguration(TezRuntimeConfiguration - .TEZ_RUNTIME_TRANSFER_DATA_VIA_EVENTS_SUPPORT_IN_MEM_FILE, + .TEZ_RUNTIME_TRANSFER_DATA_VIA_EVENTS_SUPPORT_IN_MEM_FILE, "false") .setAdditionalConfiguration(TezRuntimeConfiguration .TEZ_RUNTIME_PIPELINED_SHUFFLE_ENABLED, "true") @@ -129,9 +129,9 @@ public void testSetters() { assertEquals(2222, conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES, 0)); assertEquals(true, - conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED, false)); + conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_EMPTY_PARTITION_INFO_VIA_EVENTS_ENABLED, false)); assertEquals(5120, - conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_TRANSFER_DATA_VIA_EVENTS_MAX_SIZE, 512)); + conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_TRANSFER_DATA_VIA_EVENTS_MAX_SIZE, 512)); assertEquals(false, conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_TRANSFER_DATA_VIA_EVENTS_SUPPORT_IN_MEM_FILE, true)); assertEquals("io", conf.get("io.shouldExist")); diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/input/TestOrderedGroupedKVInput.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/input/TestOrderedGroupedKVInput.java index c1cdf7cfa0..ce709a2d94 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/input/TestOrderedGroupedKVInput.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/input/TestOrderedGroupedKVInput.java @@ -55,7 +55,6 @@ public void testInterruptWhileAwaitingInput() throws IOException, TezException { } catch (IOException e) { Assert.assertTrue(e instanceof IOInterruptedException); } - } @Test @@ -72,7 +71,6 @@ public void testMergeConfig() throws IOException, TezException { String[] workingDirs = new String[]{"workDir1"}; TezCounters counters = new TezCounters(); - doReturn(payLoad).when(inputContext).getUserPayload(); doReturn(workingDirs).when(inputContext).getWorkDirs(); doReturn(counters).when(inputContext).getCounters(); @@ -86,7 +84,6 @@ public void testMergeConfig() throws IOException, TezException { assertEquals("local-value", mergedConf.get("local-key")); } - private InputContext createMockInputContext() throws IOException { InputContext inputContext = mock(InputContext.class); Configuration conf = new TezConfiguration(); @@ -94,7 +91,6 @@ private InputContext createMockInputContext() throws IOException { String[] workingDirs = new String[]{"workDir1"}; TezCounters counters = new TezCounters(); - doReturn(payLoad).when(inputContext).getUserPayload(); doReturn(workingDirs).when(inputContext).getWorkDirs(); doReturn(200 * 1024 * 1024l).when(inputContext).getTotalMemoryAvailableToTask(); @@ -139,5 +135,4 @@ Shuffle createShuffle() throws IOException { return shuffle; } } - } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/input/TestSortedGroupedMergedInput.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/input/TestSortedGroupedMergedInput.java index c8ae67b95c..f1828296af 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/input/TestSortedGroupedMergedInput.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/input/TestSortedGroupedMergedInput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -44,17 +44,17 @@ public class TestSortedGroupedMergedInput { MergedInputContext createMergedInputContext() { return mock(MergedInputContext.class); } - + @Test(timeout = 5000) public void testSimple() throws Exception { - SortedTestKeyValuesReader kvsReader1 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, - new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } }); + SortedTestKeyValuesReader kvsReader1 = new SortedTestKeyValuesReader(new int[]{1, 2, 3}, + new int[][]{{1, 1}, {2, 2}, {3, 3}}); - SortedTestKeyValuesReader kvsReader2 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, - new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } }); + SortedTestKeyValuesReader kvsReader2 = new SortedTestKeyValuesReader(new int[]{1, 2, 3}, + new int[][]{{1, 1}, {2, 2}, {3, 3}}); - SortedTestKeyValuesReader kvsReader3 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, - new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } }); + SortedTestKeyValuesReader kvsReader3 = new SortedTestKeyValuesReader(new int[]{1, 2, 3}, + new int[][]{{1, 1}, {2, 2}, {3, 3}}); SortedTestInput sInput1 = new SortedTestInput(kvsReader1); SortedTestInput sInput2 = new SortedTestInput(kvsReader2); @@ -92,7 +92,7 @@ private void getNextFromFinishedReader(KeyValuesReader kvsReader) { try { boolean hasNext = kvsReader.next(); fail(); - } catch(IOException e) { + } catch (IOException e) { assertTrue(e.getMessage().contains("For usage, please refer to")); } } @@ -100,15 +100,14 @@ private void getNextFromFinishedReader(KeyValuesReader kvsReader) { @Test(timeout = 5000) public void testSkippedKey() throws Exception { + SortedTestKeyValuesReader kvsReader1 = new SortedTestKeyValuesReader(new int[]{1, 2, 3}, + new int[][]{{1, 1}, {2, 2}, {3, 3}}); - SortedTestKeyValuesReader kvsReader1 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, - new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } }); - - SortedTestKeyValuesReader kvsReader2 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, - new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } }); + SortedTestKeyValuesReader kvsReader2 = new SortedTestKeyValuesReader(new int[]{1, 2, 3}, + new int[][]{{1, 1}, {2, 2}, {3, 3}}); - SortedTestKeyValuesReader kvsReader3 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, - new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } }); + SortedTestKeyValuesReader kvsReader3 = new SortedTestKeyValuesReader(new int[]{1, 2, 3}, + new int[][]{{1, 1}, {2, 2}, {3, 3}}); SortedTestInput sInput1 = new SortedTestInput(kvsReader1); SortedTestInput sInput2 = new SortedTestInput(kvsReader2); @@ -146,14 +145,14 @@ public void testSkippedKey() throws Exception { @Test(timeout = 5000) public void testPartialValuesSkip() throws Exception { - SortedTestKeyValuesReader kvsReader1 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, - new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } }); + SortedTestKeyValuesReader kvsReader1 = new SortedTestKeyValuesReader(new int[]{1, 2, 3}, + new int[][]{{1, 1}, {2, 2}, {3, 3}}); - SortedTestKeyValuesReader kvsReader2 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, - new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } }); + SortedTestKeyValuesReader kvsReader2 = new SortedTestKeyValuesReader(new int[]{1, 2, 3}, + new int[][]{{1, 1}, {2, 2}, {3, 3}}); - SortedTestKeyValuesReader kvsReader3 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, - new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } }); + SortedTestKeyValuesReader kvsReader3 = new SortedTestKeyValuesReader(new int[]{1, 2, 3}, + new int[][]{{1, 1}, {2, 2}, {3, 3}}); SortedTestInput sInput1 = new SortedTestInput(kvsReader1); SortedTestInput sInput2 = new SortedTestInput(kvsReader2); @@ -193,14 +192,14 @@ public void testPartialValuesSkip() throws Exception { @Test(timeout = 5000) public void testOrdering() throws Exception { - SortedTestKeyValuesReader kvsReader1 = new SortedTestKeyValuesReader(new int[] { 2, 4 }, - new int[][] { { 2, 2 }, { 4, 4 } }); + SortedTestKeyValuesReader kvsReader1 = new SortedTestKeyValuesReader(new int[]{2, 4}, + new int[][]{{2, 2}, {4, 4}}); - SortedTestKeyValuesReader kvsReader2 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, - new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } }); + SortedTestKeyValuesReader kvsReader2 = new SortedTestKeyValuesReader(new int[]{1, 2, 3}, + new int[][]{{1, 1}, {2, 2}, {3, 3}}); - SortedTestKeyValuesReader kvsReader3 = new SortedTestKeyValuesReader(new int[] { 4, 5, 6, 7 }, - new int[][] { { 4, 4 }, { 5, 5 }, { 6, 6 }, { 7, 7 } }); + SortedTestKeyValuesReader kvsReader3 = new SortedTestKeyValuesReader(new int[]{4, 5, 6, 7}, + new int[][]{{4, 4}, {5, 5}, {6, 6}, {7, 7}}); SortedTestInput sInput1 = new SortedTestInput(kvsReader1); SortedTestInput sInput2 = new SortedTestInput(kvsReader2); @@ -245,14 +244,14 @@ public void testOrdering() throws Exception { @Test(timeout = 5000) public void testSkippedKey2() throws Exception { - SortedTestKeyValuesReader kvsReader1 = new SortedTestKeyValuesReader(new int[] { 2, 4 }, - new int[][] { { 2, 2 }, { 4, 4 } }); + SortedTestKeyValuesReader kvsReader1 = new SortedTestKeyValuesReader(new int[]{2, 4}, + new int[][]{{2, 2}, {4, 4}}); - SortedTestKeyValuesReader kvsReader2 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, - new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } }); + SortedTestKeyValuesReader kvsReader2 = new SortedTestKeyValuesReader(new int[]{1, 2, 3}, + new int[][]{{1, 1}, {2, 2}, {3, 3}}); - SortedTestKeyValuesReader kvsReader3 = new SortedTestKeyValuesReader(new int[] { 4, 5, 6, 7 }, - new int[][] { { 4, 4 }, { 5, 5 }, { 6, 6 }, { 7, 7 } }); + SortedTestKeyValuesReader kvsReader3 = new SortedTestKeyValuesReader(new int[]{4, 5, 6, 7}, + new int[][]{{4, 4}, {5, 5}, {6, 6}, {7, 7}}); SortedTestInput sInput1 = new SortedTestInput(kvsReader1); SortedTestInput sInput2 = new SortedTestInput(kvsReader2); @@ -296,19 +295,19 @@ public void testSkippedKey2() throws Exception { } getNextFromFinishedReader(kvsReader); } - + // Reads all values for a key, but doesn't trigger the last hasNext() call. @Test(timeout = 5000) public void testSkippedKey3() throws Exception { - SortedTestKeyValuesReader kvsReader1 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3, 4 }, - new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 }, {4, 4} }); + SortedTestKeyValuesReader kvsReader1 = new SortedTestKeyValuesReader(new int[]{1, 2, 3, 4}, + new int[][]{{1, 1}, {2, 2}, {3, 3}, {4, 4}}); - SortedTestKeyValuesReader kvsReader2 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3, 4 }, - new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 }, {4, 4} }); + SortedTestKeyValuesReader kvsReader2 = new SortedTestKeyValuesReader(new int[]{1, 2, 3, 4}, + new int[][]{{1, 1}, {2, 2}, {3, 3}, {4, 4}}); - SortedTestKeyValuesReader kvsReader3 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3, 4 }, - new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 }, {4, 4} }); + SortedTestKeyValuesReader kvsReader3 = new SortedTestKeyValuesReader(new int[]{1, 2, 3, 4}, + new int[][]{{1, 1}, {2, 2}, {3, 3}, {4, 4}}); SortedTestInput sInput1 = new SortedTestInput(kvsReader1); SortedTestInput sInput2 = new SortedTestInput(kvsReader2); @@ -348,14 +347,14 @@ public void testSkippedKey3() throws Exception { @Test(timeout = 5000) public void testEmptySources() throws Exception { - SortedTestKeyValuesReader kvsReader1 = new SortedTestKeyValuesReader(new int[] {}, - new int[][] {}); + SortedTestKeyValuesReader kvsReader1 = new SortedTestKeyValuesReader(new int[]{}, + new int[][]{}); - SortedTestKeyValuesReader kvsReader2 = new SortedTestKeyValuesReader(new int[] {}, - new int[][] {}); + SortedTestKeyValuesReader kvsReader2 = new SortedTestKeyValuesReader(new int[]{}, + new int[][]{}); - SortedTestKeyValuesReader kvsReader3 = new SortedTestKeyValuesReader(new int[] {}, - new int[][] {}); + SortedTestKeyValuesReader kvsReader3 = new SortedTestKeyValuesReader(new int[]{}, + new int[][]{}); SortedTestInput sInput1 = new SortedTestInput(kvsReader1); SortedTestInput sInput2 = new SortedTestInput(kvsReader2); @@ -402,14 +401,14 @@ public void testSimpleConcatenatedMergedKeyValueInput() throws Exception { @Test(timeout = 5000) public void testSimpleConcatenatedMergedKeyValuesInput() throws Exception { - SortedTestKeyValuesReader kvsReader1 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, - new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } }); + SortedTestKeyValuesReader kvsReader1 = new SortedTestKeyValuesReader(new int[]{1, 2, 3}, + new int[][]{{1, 1}, {2, 2}, {3, 3}}); - SortedTestKeyValuesReader kvsReader2 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, - new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } }); + SortedTestKeyValuesReader kvsReader2 = new SortedTestKeyValuesReader(new int[]{1, 2, 3}, + new int[][]{{1, 1}, {2, 2}, {3, 3}}); - SortedTestKeyValuesReader kvsReader3 = new SortedTestKeyValuesReader(new int[] { 1, 2, 3 }, - new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 } }); + SortedTestKeyValuesReader kvsReader3 = new SortedTestKeyValuesReader(new int[]{1, 2, 3}, + new int[][]{{1, 1}, {2, 2}, {3, 3}}); SortedTestInput sInput1 = new SortedTestInput(kvsReader1); SortedTestInput sInput2 = new SortedTestInput(kvsReader2); @@ -446,7 +445,7 @@ private void getNextFromFinishedReader(KeyValueReader kvReader) { try { boolean hasNext = kvReader.next(); fail(); - } catch(IOException e) { + } catch (IOException e) { assertTrue(e.getMessage().contains("For usage, please refer to")); } } @@ -566,7 +565,6 @@ public Object getCurrentValue() throws IOException { } } - private static class RawComparatorForTest implements RawComparator { @Override @@ -579,5 +577,4 @@ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { throw new UnsupportedOperationException(); } } - } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/OutputTestHelpers.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/OutputTestHelpers.java index 573b3e9a18..b749013a02 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/OutputTestHelpers.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/OutputTestHelpers.java @@ -58,7 +58,8 @@ static OutputContext createOutputContext(Configuration conf, Configuration userP throws IOException { OutputContext ctx = mock(OutputContext.class); doAnswer(new Answer() { - @Override public Void answer(InvocationOnMock invocation) throws Throwable { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { long requestedSize = (Long) invocation.getArguments()[0]; MemoryUpdateCallbackHandler callback = (MemoryUpdateCallbackHandler) invocation .getArguments()[1]; @@ -71,7 +72,7 @@ static OutputContext createOutputContext(Configuration conf, Configuration userP doReturn("taskVertex").when(ctx).getTaskVertexName(); doReturn("destinationVertex").when(ctx).getDestinationVertexName(); doReturn("UUID").when(ctx).getUniqueIdentifier(); - doReturn(new String[] { workingDir.toString() }).when(ctx).getWorkDirs(); + doReturn(new String[]{workingDir.toString()}).when(ctx).getWorkDirs(); doReturn(200 * 1024 * 1024l).when(ctx).getTotalMemoryAvailableToTask(); doReturn(new TezCounters()).when(ctx).getCounters(); OutputStatisticsReporter statsReporter = mock(OutputStatisticsReporter.class); diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java index 47f841c3d6..4d548e912a 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java @@ -73,7 +73,7 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; -@SuppressWarnings({ "rawtypes", "unchecked" }) +@SuppressWarnings({"rawtypes", "unchecked"}) @RunWith(Parameterized.class) public class TestOnFileSortedOutput { private static final Random rnd = new Random(); @@ -89,7 +89,7 @@ public class TestOnFileSortedOutput { //For sorter (pipelined / Default) private SorterImpl sorterImpl; private int sorterThreads; - + final AtomicLong outputSize = new AtomicLong(); final AtomicLong numRecords = new AtomicLong(); @@ -104,13 +104,13 @@ public class TestOnFileSortedOutput { * Constructor * * @param sendEmptyPartitionViaEvent - * @param sorterImpl Which sorter impl ( pipeline/legacy ) - * @param sorterThreads number of threads needed for sorter (required only for pipelined sorter) - * @param emptyPartitionIdx for which data should not be generated + * @param sorterImpl Which sorter impl ( pipeline/legacy ) + * @param sorterThreads number of threads needed for sorter (required only for pipelined sorter) + * @param emptyPartitionIdx for which data should not be generated */ public TestOnFileSortedOutput(boolean sendEmptyPartitionViaEvent, - SorterImpl sorterImpl, int sorterThreads, int emptyPartitionIdx, - ReportPartitionStats reportPartitionStats) throws IOException { + SorterImpl sorterImpl, int sorterThreads, int emptyPartitionIdx, + ReportPartitionStats reportPartitionStats) throws IOException { this.sendEmptyPartitionViaEvent = sendEmptyPartitionViaEvent; this.emptyPartitionIdx = emptyPartitionIdx; this.sorterImpl = sorterImpl; @@ -156,28 +156,28 @@ public static Collection getParameters() { Collection parameters = new ArrayList(); //empty_partition_via_events_enabled, noOfSortThreads, // partitionToBeEmpty, reportPartitionStats - parameters.add(new Object[] { false, SorterImpl.LEGACY, 1, -1, - ReportPartitionStats.ENABLED }); - parameters.add(new Object[] { false, SorterImpl.LEGACY, 1, 0, - ReportPartitionStats.ENABLED }); - parameters.add(new Object[] { true, SorterImpl.LEGACY, 1, -1, - ReportPartitionStats.ENABLED }); - parameters.add(new Object[] { true, SorterImpl.LEGACY, 1, 0, - ReportPartitionStats.ENABLED }); - parameters.add(new Object[] { true, SorterImpl.LEGACY, 1, 0, - ReportPartitionStats.PRECISE }); + parameters.add(new Object[]{false, SorterImpl.LEGACY, 1, -1, + ReportPartitionStats.ENABLED}); + parameters.add(new Object[]{false, SorterImpl.LEGACY, 1, 0, + ReportPartitionStats.ENABLED}); + parameters.add(new Object[]{true, SorterImpl.LEGACY, 1, -1, + ReportPartitionStats.ENABLED}); + parameters.add(new Object[]{true, SorterImpl.LEGACY, 1, 0, + ReportPartitionStats.ENABLED}); + parameters.add(new Object[]{true, SorterImpl.LEGACY, 1, 0, + ReportPartitionStats.PRECISE}); //Pipelined sorter - parameters.add(new Object[] { false, SorterImpl.PIPELINED, 2, -1, - ReportPartitionStats.ENABLED }); - parameters.add(new Object[] { false, SorterImpl.PIPELINED, 2, 0, - ReportPartitionStats.ENABLED }); - parameters.add(new Object[] { true, SorterImpl.PIPELINED, 2, -1, - ReportPartitionStats.ENABLED }); - parameters.add(new Object[] { true, SorterImpl.PIPELINED, 2, 0, - ReportPartitionStats.ENABLED }); - parameters.add(new Object[] { true, SorterImpl.PIPELINED, 2, 0, - ReportPartitionStats.PRECISE }); + parameters.add(new Object[]{false, SorterImpl.PIPELINED, 2, -1, + ReportPartitionStats.ENABLED}); + parameters.add(new Object[]{false, SorterImpl.PIPELINED, 2, 0, + ReportPartitionStats.ENABLED}); + parameters.add(new Object[]{true, SorterImpl.PIPELINED, 2, -1, + ReportPartitionStats.ENABLED}); + parameters.add(new Object[]{true, SorterImpl.PIPELINED, 2, 0, + ReportPartitionStats.ENABLED}); + parameters.add(new Object[]{true, SorterImpl.PIPELINED, 2, 0, + ReportPartitionStats.PRECISE}); return parameters; } @@ -210,12 +210,12 @@ private void _testPipelinedShuffle(String sorterName) throws Exception { assertTrue(sortedOutput.pipelinedShuffle); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testPipelinedShuffle() throws Exception { _testPipelinedShuffle(SorterImpl.PIPELINED.name()); } - @Test (timeout = 5000) + @Test(timeout = 5000) public void testPipelinedShuffleWithFinalMerge() throws Exception { conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 3); conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_SORTER_CLASS, SorterImpl.PIPELINED.name()); @@ -252,14 +252,13 @@ public void testPipelinedSettingsWithDefaultSorter() throws Exception { sortedOutput.start(); fail("Should have thrown illegal argument exception as pipelining is enabled with " + "DefaultSorter"); - } catch(IllegalArgumentException ie) { + } catch (IllegalArgumentException ie) { assertTrue(ie.getMessage().contains("works with PipelinedSorter")); } - } - @Test (timeout = 5000) - public void testSortBufferSize() throws Exception{ + @Test(timeout = 5000) + public void testSortBufferSize() throws Exception { OutputContext context = createTezOutputContext(); conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 2048); UserPayload payLoad = TezUtils.createUserPayloadFromConf(conf); @@ -268,9 +267,9 @@ public void testSortBufferSize() throws Exception{ try { //Memory limit checks are done in sorter impls. For e.g, defaultsorter does not support > 2GB sortedOutput.initialize(); - DefaultSorter sorter = new DefaultSorter(context, conf, 100, 3500*1024*1024l); + DefaultSorter sorter = new DefaultSorter(context, conf, 100, 3500 * 1024 * 1024l); fail(); - } catch(IllegalArgumentException e) { + } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB)); } conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 0); @@ -280,7 +279,7 @@ public void testSortBufferSize() throws Exception{ try { sortedOutput.initialize(); fail(); - } catch(IllegalArgumentException e) { + } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB)); } } @@ -375,29 +374,30 @@ public void testAllEmptyPartition() throws Exception { } private OutputContext createTezOutputContext() throws IOException { - String[] workingDirs = { workingDir.toString() }; + String[] workingDirs = {workingDir.toString()}; Configuration localConf = new Configuration(false); UserPayload payLoad = TezUtils.createUserPayloadFromConf(conf); DataOutputBuffer serviceProviderMetaData = new DataOutputBuffer(); serviceProviderMetaData.writeInt(PORT); TezCounters counters = new TezCounters(); - + OutputStatisticsReporter reporter = mock(OutputStatisticsReporter.class); doAnswer(new Answer() { - @Override public Object answer(InvocationOnMock invocation) throws Throwable { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { outputSize.set((Long) invocation.getArguments()[0]); return null; } }).when(reporter).reportDataSize(anyLong()); doAnswer(new Answer() { - @Override public Object answer(InvocationOnMock invocation) throws Throwable { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { numRecords.set((Long) invocation.getArguments()[0]); return null; } }).when(reporter).reportItemsProcessed(anyLong()); - OutputContext context = mock(OutputContext.class); doReturn(localConf).when(context).getContainerConfiguration(); doReturn(counters).when(context).getCounters(); @@ -412,7 +412,8 @@ private OutputContext createTezOutputContext() throws IOException { (conf.get(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID, TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID_DEFAULT)); doAnswer(new Answer() { - @Override public Object answer(InvocationOnMock invocation) throws Throwable { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { long requestedSize = (Long) invocation.getArguments()[0]; MemoryUpdateCallbackHandler callback = (MemoryUpdateCallbackHandler) invocation .getArguments()[1]; @@ -427,7 +428,7 @@ private OutputContext createTezOutputContext() throws IOException { return context; } - @Test(timeout=5000) + @Test(timeout = 5000) public void testInvalidSorter() throws Exception { try { _testPipelinedShuffle("Foo"); @@ -437,10 +438,8 @@ public void testInvalidSorter() throws Exception { } } - @Test(timeout=5000) + @Test(timeout = 5000) public void testLowerCaseNamedSorter() throws Exception { _testPipelinedShuffle("Pipelined"); } - - } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java index cdb246d069..4f1dd876e3 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileUnorderedKVOutput.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -146,7 +146,7 @@ public void testGeneratedDataMovementEvent() throws Exception { assertEquals(45, task.getTaskStatistics().getIOStatistics().values().iterator().next().getDataSize()); assertEquals(5, task.getTaskStatistics().getIOStatistics().values().iterator().next().getItemsProcessed()); assertTrue(events != null && events.size() == 2); - CompositeDataMovementEvent dmEvent = (CompositeDataMovementEvent)events.get(1); + CompositeDataMovementEvent dmEvent = (CompositeDataMovementEvent) events.get(1); assertEquals("Invalid source index", 0, dmEvent.getSourceIndexStart()); @@ -215,8 +215,7 @@ public void testWithPipelinedShuffle() throws Exception { verify(outputContext, atLeast(1)).sendEvents(eventsCaptor.capture()); events = eventsCaptor.getValue(); - - CompositeDataMovementEvent dmEvent = (CompositeDataMovementEvent)events.get(1); + CompositeDataMovementEvent dmEvent = (CompositeDataMovementEvent) events.get(1); assertEquals("Invalid source index", 0, dmEvent.getSourceIndexStart()); DataMovementEventPayloadProto shufflePayload = DataMovementEventPayloadProto @@ -231,7 +230,7 @@ public void testWithPipelinedShuffle() throws Exception { } private OutputContext createOutputContext(Configuration payloadConf, Configuration baseConf, - TezSharedExecutor sharedExecutor) throws IOException { + TezSharedExecutor sharedExecutor) throws IOException { int appAttemptNumber = 1; TezUmbilical tezUmbilical = mock(TezUmbilical.class); String dagName = "currentDAG"; @@ -242,7 +241,7 @@ private OutputContext createOutputContext(Configuration payloadConf, Configurati TezTaskID taskID = TezTaskID.getInstance(vertexID, 1); TezTaskAttemptID taskAttemptID = TezTaskAttemptID.getInstance(taskID, 1); UserPayload userPayload = TezUtils.createUserPayloadFromConf(payloadConf); - + TaskSpec mockSpec = mock(TaskSpec.class); when(mockSpec.getInputs()).thenReturn(Collections.singletonList(mock(InputSpec.class))); when(mockSpec.getOutputs()).thenReturn(Collections.singletonList(mock(OutputSpec.class))); @@ -251,7 +250,7 @@ private OutputContext createOutputContext(Configuration payloadConf, Configurati new DefaultHadoopShim(), sharedExecutor); LogicalIOProcessorRuntimeTask runtimeTask = spy(task); - + Map auxEnv = new HashMap(); ByteBuffer bb = ByteBuffer.allocate(4); bb.putInt(shufflePort); @@ -259,11 +258,10 @@ private OutputContext createOutputContext(Configuration payloadConf, Configurati AuxiliaryServiceHelper.setServiceDataIntoEnv(payloadConf.get(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID, TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID_DEFAULT), bb, auxEnv); - OutputDescriptor outputDescriptor = mock(OutputDescriptor.class); when(outputDescriptor.getClassName()).thenReturn("OutputDescriptor"); - OutputContext realOutputContext = new TezOutputContextImpl(baseConf, new String[] {workDir.toString()}, + OutputContext realOutputContext = new TezOutputContextImpl(baseConf, new String[]{workDir.toString()}, appAttemptNumber, tezUmbilical, dagName, taskVertexName, destinationVertexName, -1, taskAttemptID, 0, userPayload, runtimeTask, null, auxEnv, new MemoryDistributor(1, 1, payloadConf), outputDescriptor, null, @@ -274,7 +272,8 @@ null, auxEnv, new MemoryDistributor(1, 1, payloadConf), outputDescriptor, null, Assert.assertTrue(task.getTaskStatistics().getIOStatistics().containsKey(destinationVertexName)); OutputContext outputContext = spy(realOutputContext); doAnswer(new Answer() { - @Override public Object answer(InvocationOnMock invocation) throws Throwable { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { long requestedSize = (Long) invocation.getArguments()[0]; MemoryUpdateCallbackHandler callback = (MemoryUpdateCallbackHandler) invocation .getArguments()[1]; diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOrderedPartitionedKVOutput2.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOrderedPartitionedKVOutput2.java index 29ce890309..cffb66f035 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOrderedPartitionedKVOutput2.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOrderedPartitionedKVOutput2.java @@ -89,7 +89,7 @@ public void testNonStartedOutput() throws IOException { .getEmptyPartitions()); BitSet emptyPartionsBitSet = TezUtilsInternal.fromByteArray(emptyPartitions); assertEquals(numPartitions, emptyPartionsBitSet.cardinality()); - for (int i = 0 ; i < numPartitions ; i++) { + for (int i = 0; i < numPartitions; i++) { assertTrue(emptyPartionsBitSet.get(i)); } } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestUnorderedKVOutput2.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestUnorderedKVOutput2.java index a52788e716..e194b29b2e 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestUnorderedKVOutput2.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestUnorderedKVOutput2.java @@ -55,7 +55,7 @@ public void setup() throws IOException { workingDir = new Path(System.getProperty("test.build.data", System.getProperty("java.io.tmpdir", "/tmp")), TestUnorderedKVOutput2.class.getName()).makeQualified( - localFs.getUri(), localFs.getWorkingDirectory()); + localFs.getUri(), localFs.getWorkingDirectory()); conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, Text.class.getName()); conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, Text.class.getName()); conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS, @@ -88,7 +88,7 @@ public void testNonStartedOutput() throws Exception { .getEmptyPartitions()); BitSet emptyPartionsBitSet = TezUtilsInternal.fromByteArray(emptyPartitions); assertEquals(numPartitions, emptyPartionsBitSet.cardinality()); - for (int i = 0 ; i < numPartitions ; i++) { + for (int i = 0; i < numPartitions; i++) { assertTrue(emptyPartionsBitSet.get(i)); } } diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestUnorderedPartitionedKVOutput2.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestUnorderedPartitionedKVOutput2.java index 52e06300dd..b794119015 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestUnorderedPartitionedKVOutput2.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestUnorderedPartitionedKVOutput2.java @@ -35,7 +35,6 @@ // Tests which don't require parameterization public class TestUnorderedPartitionedKVOutput2 { - @Test(timeout = 5000) public void testNonStartedOutput() throws Exception { OutputContext outputContext = OutputTestHelpers.createOutputContext(); diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/testutils/KVDataGen.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/testutils/KVDataGen.java index 960d371078..8dff4bfe75 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/testutils/KVDataGen.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/testutils/KVDataGen.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -97,4 +97,4 @@ public IntWritable getvalue() { return this.value; } } -} \ No newline at end of file +} diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/testutils/RuntimeTestUtils.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/testutils/RuntimeTestUtils.java index 0885178ee5..942187a6b3 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/testutils/RuntimeTestUtils.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/testutils/RuntimeTestUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/utils/TestCodecUtils.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/utils/TestCodecUtils.java index 5141598200..2239f62bbc 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/utils/TestCodecUtils.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/utils/TestCodecUtils.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java index 5c99f3efbf..5ce4ab0017 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,6 +21,7 @@ import java.io.IOException; import java.nio.ByteBuffer; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileSystem; @@ -107,10 +108,10 @@ public void run() throws Exception { boolean doLocalityCheck = getContext().getUserPayload().getPayload().get(0) > 0 ? true : false; int broadcastSum = getContext().getUserPayload().getPayload().get(1); int expectedSum = broadcastSum + getContext().getTaskIndex(); - System.out.println("Index: " + getContext().getTaskIndex() + + System.out.println("Index: " + getContext().getTaskIndex() + " sum: " + sum + " expectedSum: " + expectedSum + " broadcastSum: " + broadcastSum); - Preconditions.checkState((sum == expectedSum), "Sum = " + sum); - + Preconditions.checkState((sum == expectedSum), "Sum = " + sum); + if (doLocalityCheck) { ObjectRegistry objectRegistry = getContext().getObjectRegistry(); String index = (String) objectRegistry.get(String.valueOf(getContext().getTaskIndex())); @@ -122,11 +123,10 @@ public void run() throws Exception { } } } - } private DAG createDAG(FileSystem fs, TezConfiguration tezConf, - Path stagingDir, boolean doLocalityCheck) throws IOException, YarnException { + Path stagingDir, boolean doLocalityCheck) throws IOException, YarnException { int numBroadcastTasks = 2; int numOneToOneTasks = 3; @@ -149,7 +149,7 @@ private DAG createDAG(FileSystem fs, TezConfiguration tezConf, Vertex broadcastVertex = Vertex.create("Broadcast", ProcessorDescriptor.create( InputProcessor.class.getName()), numBroadcastTasks); - + Vertex inputVertex = Vertex.create("Input", ProcessorDescriptor.create( InputProcessor.class.getName()).setUserPayload(procPayload), numOneToOneTasks); @@ -174,7 +174,7 @@ private DAG createDAG(FileSystem fs, TezConfiguration tezConf, edgeConf.createDefaultBroadcastEdgeProperty())); return dag; } - + public boolean run(Configuration conf, boolean doLocalityCheck) throws Exception { System.out.println("Running BroadcastAndOneToOneExample"); // conf and UGI @@ -191,11 +191,11 @@ public boolean run(Configuration conf, boolean doLocalityCheck) throws Exception FileSystem fs = FileSystem.get(tezConf); String stagingDirStr = tezConf.get(TezConfiguration.TEZ_AM_STAGING_DIR, TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT) + Path.SEPARATOR + - "BroadcastAndOneToOneExample" + Path.SEPARATOR + Long.toString(System.currentTimeMillis()); + "BroadcastAndOneToOneExample" + Path.SEPARATOR + Long.toString(System.currentTimeMillis()); Path stagingDir = new Path(stagingDirStr); tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirStr); stagingDir = fs.makeQualified(stagingDir); - + // No need to add jar containing this class as assumed to be part of // the tez jars. @@ -209,24 +209,24 @@ public boolean run(Configuration conf, boolean doLocalityCheck) throws Exception DAGClient dagClient = null; try { - DAG dag = createDAG(fs, tezConf, stagingDir, doLocalityCheck); + DAG dag = createDAG(fs, tezConf, stagingDir, doLocalityCheck); - tezSession.waitTillReady(); - dagClient = tezSession.submitDAG(dag); + tezSession.waitTillReady(); + dagClient = tezSession.submitDAG(dag); - // monitoring - DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(null); - if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) { - System.out.println("DAG diagnostics: " + dagStatus.getDiagnostics()); - return false; - } - return true; + // monitoring + DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(null); + if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) { + System.out.println("DAG diagnostics: " + dagStatus.getDiagnostics()); + return false; + } + return true; } finally { fs.delete(stagingDir, true); tezSession.stop(); } } - + @Override public int run(String[] args) throws Exception { boolean doLocalityCheck = true; @@ -252,12 +252,12 @@ public int run(String[] args) throws Exception { boolean status = run(getConf(), doLocalityCheck); return status ? 0 : 1; } - + private static void printUsage() { System.err.println("broadcastAndOneToOneExample " + skipLocalityCheck); ToolRunner.printGenericCommandUsage(System.err); } - + static String skipLocalityCheck = "-skipLocalityCheck"; public static void main(String[] args) throws Exception { diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastLoadGen.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastLoadGen.java index 03bb0519a0..3e72cde2bc 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastLoadGen.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastLoadGen.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -69,7 +69,7 @@ public void run() throws Exception { UnorderedKVOutput output = (UnorderedKVOutput) out; KeyValueWriter kvWriter = output.getWriter(); int approxNumInts = bytesToGenerate / 6; - for (int i = 0 ; i < approxNumInts ; i++) { + for (int i = 0; i < approxNumInts; i++) { kvWriter.write(NullWritable.get(), new IntWritable(random.nextInt())); } } @@ -107,7 +107,7 @@ private DAG createDAG(int numGenTasks, int totalSourceDataSize, int numFetcherTa Vertex fetchVertex = Vertex.create("FetchVertex", ProcessorDescriptor.create(InputFetchProcessor.class.getName()), numFetcherTasks); UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig.newBuilder(NullWritable.class - .getName(), IntWritable.class.getName()).setCompression(false, null, null).build(); + .getName(), IntWritable.class.getName()).setCompression(false, null, null).build(); DAG dag = DAG.create("BroadcastLoadGen"); dag.addVertex(broadcastVertex).addVertex(fetchVertex).addEdge( diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/CartesianProduct.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/CartesianProduct.java index aea662ae04..9320e4e7ff 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/CartesianProduct.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/CartesianProduct.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -89,7 +89,7 @@ public class CartesianProduct extends TezExampleBase { private static final String VERTEX2 = "Vertex2"; private static final String VERTEX3 = "Vertex3"; private static final Logger LOG = LoggerFactory.getLogger(CartesianProduct.class); - private static final String[] sourceVertices = new String[] {VERTEX1, VERTEX2}; + private static final String[] sourceVertices = new String[]{VERTEX1, VERTEX2}; public static class TokenProcessor extends SimpleProcessor { public TokenProcessor(ProcessorContext context) { @@ -105,7 +105,7 @@ public void run() throws Exception { while (kvReader.next()) { Object key = kvReader.getCurrentKey(); Object value = kvReader.getCurrentValue(); - kvWriter.write(new Text((String)key), new IntWritable(1)); + kvWriter.write(new Text((String) key), new IntWritable(1)); } } } @@ -124,11 +124,11 @@ public void run() throws Exception { Set rightSet = new HashSet<>(); while (kvReader1.next()) { - Text key = (Text)(kvReader1.getCurrentKey()); + Text key = (Text) (kvReader1.getCurrentKey()); leftSet.add(new Text(key)); } while (kvReader2.next()) { - Text key = (Text)(kvReader2.getCurrentKey()); + Text key = (Text) (kvReader2.getCurrentKey()); rightSet.add(new Text(key)); } @@ -218,7 +218,7 @@ public Reader getReader() throws Exception { public boolean next() throws IOException { if (i == -1) { for (int j = 0; j < numRecordPerSrc; j++) { - keys[j] = ""+j; + keys[j] = "" + j; } } i++; @@ -324,9 +324,9 @@ public void write(Object key, Object value) throws IOException { private DAG createDAG(TezConfiguration tezConf) throws IOException { InputDescriptor inputDescriptor = InputDescriptor.create(FakeInput.class.getName()); InputInitializerDescriptor inputInitializerDescriptor = - InputInitializerDescriptor.create(FakeInputInitializer.class.getName()); + InputInitializerDescriptor.create(FakeInputInitializer.class.getName()); DataSourceDescriptor dataSourceDescriptor = - DataSourceDescriptor.create(inputDescriptor, inputInitializerDescriptor, null); + DataSourceDescriptor.create(inputDescriptor, inputInitializerDescriptor, null); Vertex v1 = Vertex.create(VERTEX1, ProcessorDescriptor.create(TokenProcessor.class.getName())); v1.addDataSource(INPUT, dataSourceDescriptor); @@ -335,30 +335,30 @@ private DAG createDAG(TezConfiguration tezConf) throws IOException { OutputDescriptor outputDescriptor = OutputDescriptor.create(FakeOutput.class.getName()); OutputCommitterDescriptor outputCommitterDescriptor = - OutputCommitterDescriptor.create(FakeOutputCommitter.class.getName()); + OutputCommitterDescriptor.create(FakeOutputCommitter.class.getName()); DataSinkDescriptor dataSinkDescriptor = - DataSinkDescriptor.create(outputDescriptor, outputCommitterDescriptor, null); + DataSinkDescriptor.create(outputDescriptor, outputCommitterDescriptor, null); CartesianProductConfig cartesianProductConfig = - new CartesianProductConfig(Arrays.asList(sourceVertices)); + new CartesianProductConfig(Arrays.asList(sourceVertices)); UserPayload userPayload = cartesianProductConfig.toUserPayload(tezConf); Vertex v3 = Vertex.create(VERTEX3, ProcessorDescriptor.create(JoinProcessor.class.getName())); v3.addDataSink(OUTPUT, dataSinkDescriptor); v3.setVertexManagerPlugin( - VertexManagerPluginDescriptor.create(CartesianProductVertexManager.class.getName()) - .setUserPayload(userPayload)); + VertexManagerPluginDescriptor.create(CartesianProductVertexManager.class.getName()) + .setUserPayload(userPayload)); EdgeManagerPluginDescriptor edgeManagerDescriptor = - EdgeManagerPluginDescriptor.create(CartesianProductEdgeManager.class.getName()); + EdgeManagerPluginDescriptor.create(CartesianProductEdgeManager.class.getName()); edgeManagerDescriptor.setUserPayload(userPayload); UnorderedPartitionedKVEdgeConfig edgeConf = - UnorderedPartitionedKVEdgeConfig.newBuilder(Text.class.getName(), IntWritable.class.getName(), - RoundRobinPartitioner.class.getName()).build(); + UnorderedPartitionedKVEdgeConfig.newBuilder(Text.class.getName(), IntWritable.class.getName(), + RoundRobinPartitioner.class.getName()).build(); EdgeProperty edgeProperty = edgeConf.createDefaultCustomEdgeProperty(edgeManagerDescriptor); return DAG.create("CrossProduct").addVertex(v1).addVertex(v2).addVertex(v3) - .addEdge(Edge.create(v1, v3, edgeProperty)).addEdge(Edge.create(v2, v3, edgeProperty)); + .addEdge(Edge.create(v1, v3, edgeProperty)).addEdge(Edge.create(v2, v3, edgeProperty)); } @Override @@ -371,7 +371,7 @@ protected int validateArgs(String[] otherArgs) { @Override protected int runJob(String[] args, TezConfiguration tezConf, - TezClient tezClient) throws Exception { + TezClient tezClient) throws Exception { DAG dag = createDAG(tezConf); return runDag(dag, isCountersLog(), LOG); } @@ -381,5 +381,3 @@ public static void main(String[] args) throws Exception { System.exit(res); } } - - diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java index cdbdf13db5..65642698bd 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/ExampleDriver.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -42,7 +42,7 @@ public class ExampleDriver { private static final DecimalFormat formatter = new DecimalFormat("###.##%"); - public static void main(String argv[]){ + public static void main(String argv[]) { int exitCode = -1; ProgramDriver pgd = new ProgramDriver(); try { @@ -54,19 +54,19 @@ public static void main(String argv[]){ "A map/reduce program that counts the words in the input files."); pgd.addClass("mapredwordcount", MapredWordCount.class, "A map/reduce program that counts the words in the input files" - + " using the mapred apis."); + + " using the mapred apis."); pgd.addClass("randomwriter", RandomWriter.class, "A map/reduce program that writes 10GB of random data per node."); pgd.addClass("randomtextwriter", RandomTextWriter.class, - "A map/reduce program that writes 10GB of random textual data per node."); + "A map/reduce program that writes 10GB of random textual data per node."); pgd.addClass("sort", Sort.class, "A map/reduce program that sorts the data written by the random" - + " writer."); + + " writer."); pgd.addClass("secondarysort", SecondarySort.class, "An example defining a secondary sort to the reduce."); pgd.addClass("join", Join.class, "A job that effects a join over sorted, equally partitioned" - + " datasets"); + + " datasets"); pgd.addClass("mrrsleep", MRRSleepJob.class, "MRR Sleep Job"); pgd.addClass("testorderedwordcount", TestOrderedWordCount.class, @@ -84,8 +84,7 @@ public static void main(String argv[]){ pgd.addClass("cartesianproduct", CartesianProduct.class, "Cartesian Product Example"); exitCode = pgd.run(argv); - } - catch(Throwable e){ + } catch (Throwable e) { e.printStackTrace(); } @@ -104,11 +103,11 @@ public static void printDAGStatus(DAGClient dagClient, String[] vertexNames) } public static void printDAGStatus(DAGClient dagClient, String[] vertexNames, - boolean displayDAGCounters, boolean displayVertexCounters) + boolean displayDAGCounters, boolean displayVertexCounters) throws IOException, TezException { Set opts = EnumSet.of(StatusGetOpts.GET_COUNTERS); DAGStatus dagStatus = dagClient.getDAGStatus( - (displayDAGCounters ? opts : null)); + (displayDAGCounters ? opts : null)); Progress progress = dagStatus.getDAGProgress(); double vProgressFloat = 0.0f; if (progress != null) { @@ -117,14 +116,14 @@ public static void printDAGStatus(DAGClient dagClient, String[] vertexNames, + dagStatus.getState() + " Progress: " + (progress.getTotalTaskCount() < 0 ? formatter.format(0.0f) : - formatter.format((double)(progress.getSucceededTaskCount()) - /progress.getTotalTaskCount()))); + formatter.format((double) (progress.getSucceededTaskCount()) + / progress.getTotalTaskCount()))); for (String vertexName : vertexNames) { VertexStatus vStatus = dagClient.getVertexStatus(vertexName, - (displayVertexCounters ? opts : null)); + (displayVertexCounters ? opts : null)); if (vStatus == null) { System.out.println("Could not retrieve status for vertex: " - + vertexName); + + vertexName); continue; } Progress vProgress = vStatus.getProgress(); @@ -133,20 +132,20 @@ public static void printDAGStatus(DAGClient dagClient, String[] vertexNames, if (vProgress.getTotalTaskCount() == 0) { vProgressFloat = 1.0f; } else if (vProgress.getTotalTaskCount() > 0) { - vProgressFloat = (double)vProgress.getSucceededTaskCount() - /vProgress.getTotalTaskCount(); + vProgressFloat = (double) vProgress.getSucceededTaskCount() + / vProgress.getTotalTaskCount(); } System.out.println("VertexStatus:" + " VertexName: " + (vertexName.equals("ivertex1") ? "intermediate-reducer" - : vertexName) + : vertexName) + " Progress: " + formatter.format(vProgressFloat)); } if (displayVertexCounters) { TezCounters counters = vStatus.getVertexCounters(); if (counters != null) { System.out.println("Vertex Counters for " + vertexName + ": " - + counters); + + counters); } } } @@ -158,6 +157,4 @@ public static void printDAGStatus(DAGClient dagClient, String[] vertexNames, } } } - } - diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java index da4d9ea76c..61ca52e0b6 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -84,13 +84,13 @@ public class FilterLinesByWord extends Configured implements Tool { private static Logger LOG = LoggerFactory.getLogger(FilterLinesByWord.class); public static final String FILTER_PARAM_NAME = "tez.runtime.examples.filterbyword.word"; - + private boolean exitOnCompletion = false; public FilterLinesByWord(boolean exitOnCompletion) { this.exitOnCompletion = exitOnCompletion; } - + private static void printUsage() { System.err.println("Usage filtelinesrbyword [-generateSplitsInClient true/]"); ToolRunner.printGenericCommandUsage(System.err); @@ -99,7 +99,7 @@ private static void printUsage() { @Override public int run(String[] args) throws Exception { Configuration conf = getConf(); - String [] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); + String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); Credentials credentials = new Credentials(); boolean generateSplitsInClient = false; @@ -158,8 +158,6 @@ public int run(String[] args) throws Exception { remoteJarStatus.getLen(), remoteJarStatus.getModificationTime()); commonLocalResources.put("dag_job.jar", dagJarLocalRsrc); - - TezClient tezSession = TezClient.create("FilterLinesByWordSession", tezConf, commonLocalResources, credentials); tezSession.start(); // Why do I need to start the TezSession. @@ -174,7 +172,7 @@ public int run(String[] args) throws Exception { UserPayload stage1Payload = TezUtils.createUserPayloadFromConf(stage1Conf); // Setup stage1 Vertex Vertex stage1Vertex = Vertex.create("stage1", ProcessorDescriptor.create( - FilterByWordInputProcessor.class.getName()).setUserPayload(stage1Payload)) + FilterByWordInputProcessor.class.getName()).setUserPayload(stage1Payload)) .addTaskLocalFiles(commonLocalResources); DataSourceDescriptor dsd; @@ -216,11 +214,11 @@ public int run(String[] args) throws Exception { LOG.info("Submitted DAG to Tez Session"); DAGStatus dagStatus = null; - String[] vNames = { "stage1", "stage2" }; + String[] vNames = {"stage1", "stage2"}; try { while (true) { dagStatus = dagClient.getDAGStatus(null); - if(dagStatus.getState() == DAGStatus.State.RUNNING || + if (dagStatus.getState() == DAGStatus.State.RUNNING || dagStatus.getState() == DAGStatus.State.SUCCEEDED || dagStatus.getState() == DAGStatus.State.FAILED || dagStatus.getState() == DAGStatus.State.KILLED || @@ -248,9 +246,8 @@ public int run(String[] args) throws Exception { return -1; } } - + dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS)); - } finally { fs.delete(stagingDir, true); tezSession.stop(); @@ -260,7 +257,7 @@ public int run(String[] args) throws Exception { LOG.info("Application completed. " + "FinalState=" + dagStatus.getState()); return dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1; } - + public static void main(String[] args) throws Exception { FilterLinesByWord fl = new FilterLinesByWord(true); int status = ToolRunner.run(new Configuration(), fl, args); diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java index 00205036f4..22d7a784f4 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -77,7 +77,7 @@ public class FilterLinesByWordOneToOne extends Configured implements Tool { public static final String FILTER_PARAM_NAME = "tez.runtime.examples.filterbyword.word"; private static void printUsage() { - System.err.println("Usage filterLinesByWordOneToOne " + System.err.println("Usage filterLinesByWordOneToOne " + " [-generateSplitsInClient true/]"); ToolRunner.printGenericCommandUsage(System.err); } @@ -112,7 +112,7 @@ public int run(String[] otherArgs) throws Exception { String inputPath = otherArgs[0]; String outputPath = otherArgs[1]; String filterWord = otherArgs[2]; - + Configuration conf = getConf(); FileSystem fs = FileSystem.get(conf); if (fs.exists(new Path(outputPath))) { @@ -144,8 +144,6 @@ public int run(String[] otherArgs) throws Exception { remoteJarStatus.getLen(), remoteJarStatus.getModificationTime()); commonLocalResources.put("dag_job.jar", dagJarLocalRsrc); - - TezClient tezSession = TezClient.create("FilterLinesByWordSession", tezConf, commonLocalResources, null); tezSession.start(); // Why do I need to start the TezSession. @@ -161,7 +159,7 @@ public int run(String[] otherArgs) throws Exception { UserPayload stage1Payload = TezUtils.createUserPayloadFromConf(stage1Conf); // Setup stage1 Vertex Vertex stage1Vertex = Vertex.create("stage1", ProcessorDescriptor.create( - FilterByWordInputProcessor.class.getName()).setUserPayload(stage1Payload)) + FilterByWordInputProcessor.class.getName()).setUserPayload(stage1Payload)) .addTaskLocalFiles(commonLocalResources); DataSourceDescriptor dsd; @@ -186,7 +184,7 @@ public int run(String[] otherArgs) throws Exception { stage2Vertex.addDataSink( "MROutput", DataSinkDescriptor.create(OutputDescriptor.create(MROutput.class.getName()) - .setUserPayload(TezUtils.createUserPayloadFromConf(stage2Conf)), + .setUserPayload(TezUtils.createUserPayloadFromConf(stage2Conf)), OutputCommitterDescriptor.create(MROutputCommitter.class.getName()), null)); UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig @@ -203,11 +201,11 @@ public int run(String[] otherArgs) throws Exception { LOG.info("Submitted DAG to Tez Session"); DAGStatus dagStatus = null; - String[] vNames = { "stage1", "stage2" }; + String[] vNames = {"stage1", "stage2"}; try { while (true) { dagStatus = dagClient.getDAGStatus(null); - if(dagStatus.getState() == DAGStatus.State.RUNNING || + if (dagStatus.getState() == DAGStatus.State.RUNNING || dagStatus.getState() == DAGStatus.State.SUCCEEDED || dagStatus.getState() == DAGStatus.State.FAILED || dagStatus.getState() == DAGStatus.State.KILLED || diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Join.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Join.java index 92be836375..05dc8934d7 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Join.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Join.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -61,14 +61,15 @@ */ public class Join extends Configured implements Tool { public static final String REDUCES_PER_HOST = "mapreduce.join.reduces_per_host"; + static int printUsage() { System.out.println("join [-r ] " + - "[-inFormat ] " + - "[-outFormat ] " + - "[-outKey ] " + - "[-outValue ] " + - "[-joinOp ] " + - "[input]* "); + "[-inFormat ] " + + "[-outFormat ] " + + "[-outKey ] " + + "[-outValue ] " + + "[-joinOp ] " + + "[input]* "); ToolRunner.printGenericCommandUsage(System.out); return 2; } @@ -87,40 +88,40 @@ public int run(String[] args) throws Exception { int num_reduces = (int) (cluster.getMaxReduceTasks() * 0.9); String join_reduces = conf.get(REDUCES_PER_HOST); if (join_reduces != null) { - num_reduces = cluster.getTaskTrackers() * - Integer.parseInt(join_reduces); + num_reduces = cluster.getTaskTrackers() * + Integer.parseInt(join_reduces); } Job job = new Job(conf); job.setJobName("join"); job.setJarByClass(Sort.class); - job.setMapperClass(Mapper.class); + job.setMapperClass(Mapper.class); job.setReducerClass(Reducer.class); - Class inputFormatClass = - SequenceFileInputFormat.class; - Class outputFormatClass = - SequenceFileOutputFormat.class; + Class inputFormatClass = + SequenceFileInputFormat.class; + Class outputFormatClass = + SequenceFileOutputFormat.class; Class outputKeyClass = BytesWritable.class; Class outputValueClass = TupleWritable.class; String op = "inner"; List otherArgs = new ArrayList(); - for(int i=0; i < args.length; ++i) { + for (int i = 0; i < args.length; ++i) { try { if ("-r".equals(args[i])) { num_reduces = Integer.parseInt(args[++i]); } else if ("-inFormat".equals(args[i])) { - inputFormatClass = - Class.forName(args[++i]).asSubclass(InputFormat.class); + inputFormatClass = + Class.forName(args[++i]).asSubclass(InputFormat.class); } else if ("-outFormat".equals(args[i])) { - outputFormatClass = - Class.forName(args[++i]).asSubclass(OutputFormat.class); + outputFormatClass = + Class.forName(args[++i]).asSubclass(OutputFormat.class); } else if ("-outKey".equals(args[i])) { - outputKeyClass = - Class.forName(args[++i]).asSubclass(WritableComparable.class); + outputKeyClass = + Class.forName(args[++i]).asSubclass(WritableComparable.class); } else if ("-outValue".equals(args[i])) { - outputValueClass = - Class.forName(args[++i]).asSubclass(Writable.class); + outputValueClass = + Class.forName(args[++i]).asSubclass(Writable.class); } else if ("-joinOp".equals(args[i])) { op = args[++i]; } else { @@ -131,7 +132,7 @@ public int run(String[] args) throws Exception { return printUsage(); } catch (ArrayIndexOutOfBoundsException except) { System.out.println("ERROR: Required parameter missing from " + - args[i-1]); + args[i - 1]); return printUsage(); // exits } } @@ -144,17 +145,17 @@ public int run(String[] args) throws Exception { return printUsage(); } - FileOutputFormat.setOutputPath(job, - new Path(otherArgs.remove(otherArgs.size() - 1))); + FileOutputFormat.setOutputPath(job, + new Path(otherArgs.remove(otherArgs.size() - 1))); List plist = new ArrayList(otherArgs.size()); for (String s : otherArgs) { plist.add(new Path(s)); } job.setInputFormatClass(CompositeInputFormat.class); - job.getConfiguration().set(CompositeInputFormat.JOIN_EXPR, - CompositeInputFormat.compose(op, inputFormatClass, - plist.toArray(new Path[0]))); + job.getConfiguration().set(CompositeInputFormat.JOIN_EXPR, + CompositeInputFormat.compose(op, inputFormatClass, + plist.toArray(new Path[0]))); job.setOutputFormatClass(outputFormatClass); job.setOutputKeyClass(outputKeyClass); @@ -162,11 +163,11 @@ public int run(String[] args) throws Exception { Date startTime = new Date(); System.out.println("Job started: " + startTime); - int ret = job.waitForCompletion(true) ? 0 : 1 ; + int ret = job.waitForCompletion(true) ? 0 : 1; Date end_time = new Date(); System.out.println("Job ended: " + end_time); - System.out.println("The job took " + - (end_time.getTime() - startTime.getTime()) /1000 + " seconds."); + System.out.println("The job took " + + (end_time.getTime() - startTime.getTime()) / 1000 + " seconds."); return ret; } @@ -174,5 +175,4 @@ public static void main(String[] args) throws Exception { int res = ToolRunner.run(getTezDecoratedConfiguration(), new Join(), args); System.exit(res); } - } diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java index 3be71315a2..a657630743 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.mapreduce.examples; @@ -101,10 +101,10 @@ public class MRRSleepJob extends Configured implements Tool { public static final String MAP_SLEEP_COUNT = "mrr.sleepjob.map.sleep.count"; public static final String REDUCE_SLEEP_COUNT = - "mrr.sleepjob.reduce.sleep.count"; + "mrr.sleepjob.reduce.sleep.count"; public static final String MAP_SLEEP_TIME = "mrr.sleepjob.map.sleep.time"; public static final String REDUCE_SLEEP_TIME = - "mrr.sleepjob.reduce.sleep.time"; + "mrr.sleepjob.reduce.sleep.time"; public static final String IREDUCE_SLEEP_COUNT = "mrr.sleepjob.ireduce.sleep.count"; public static final String IREDUCE_SLEEP_TIME = @@ -128,26 +128,29 @@ public int getPartition(IntWritable k, IntWritable v, int numPartitions) { } public static class EmptySplit extends InputSplit implements Writable { - public void write(DataOutput out) throws IOException { } - public void readFields(DataInput in) throws IOException { } - public long getLength() { return 0L; } - public String[] getLocations() { return new String[0]; } + public void write(DataOutput out) throws IOException {} + + public void readFields(DataInput in) throws IOException {} + + public long getLength() {return 0L;} + + public String[] getLocations() {return new String[0];} } public static class SleepInputFormat - extends InputFormat { + extends InputFormat { public List getSplits(JobContext jobContext) { List ret = new ArrayList(); int numSplits = jobContext.getConfiguration(). - getInt(MRJobConfig.NUM_MAPS, 1); + getInt(MRJobConfig.NUM_MAPS, 1); for (int i = 0; i < numSplits; ++i) { ret.add(new EmptySplit()); } return ret; } - public RecordReader createRecordReader( + public RecordReader createRecordReader( InputSplit ignored, TaskAttemptContext taskContext) throws IOException { Configuration conf = taskContext.getConfiguration(); @@ -159,15 +162,15 @@ public RecordReader createRecordReader( int totalIReduces = conf.getInt(IREDUCE_STAGES_COUNT, 1); - int reduceTasks = totalIReduces == 0? + int reduceTasks = totalIReduces == 0 ? taskContext.getNumReduceTasks() : - conf.getInt(IREDUCE_TASKS_COUNT, 1); - int sleepCount = totalIReduces == 0? - conf.getInt(REDUCE_SLEEP_COUNT,1) : - conf.getInt(IREDUCE_SLEEP_COUNT,1); + conf.getInt(IREDUCE_TASKS_COUNT, 1); + int sleepCount = totalIReduces == 0 ? + conf.getInt(REDUCE_SLEEP_COUNT, 1) : + conf.getInt(IREDUCE_SLEEP_COUNT, 1); final int emitPerMapTask = sleepCount * reduceTasks; - return new RecordReader() { + return new RecordReader() { private int records = 0; private int emitCount = 0; private IntWritable key = null; @@ -192,11 +195,15 @@ public boolean nextKeyValue() value.set(emit); return records++ < count; } - public IntWritable getCurrentKey() { return key; } - public IntWritable getCurrentValue() { return value; } - public void close() throws IOException { } + + public IntWritable getCurrentKey() {return key;} + + public IntWritable getCurrentValue() {return value;} + + public void close() throws IOException {} + public float getProgress() throws IOException { - return count == 0 ? 100 : records / ((float)count); + return count == 0 ? 100 : records / ((float) count); } }; } @@ -213,19 +220,19 @@ public static class SleepMapper private boolean finalAttempt = false; protected void setup(Context context) - throws IOException, InterruptedException { + throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); this.mapSleepCount = - conf.getInt(MAP_SLEEP_COUNT, mapSleepCount); + conf.getInt(MAP_SLEEP_COUNT, mapSleepCount); this.mapSleepDuration = mapSleepCount == 0 ? 0 : - conf.getLong(MAP_SLEEP_TIME , 100) / mapSleepCount; + conf.getLong(MAP_SLEEP_TIME, 100) / mapSleepCount; vertexName = conf.get( org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME); TaskAttemptID taId = context.getTaskAttemptID(); String[] taskIds = conf.getStrings(MAP_ERROR_TASK_IDS); - if (taId.getId()+1 >= context.getMaxMapAttempts()) { + if (taId.getId() + 1 >= context.getMaxMapAttempts()) { finalAttempt = true; } boolean found = false; @@ -252,7 +259,7 @@ protected void setup(Context context) } public void map(IntWritable key, IntWritable value, Context context - ) throws IOException, InterruptedException { + ) throws IOException, InterruptedException { //it is expected that every map processes mapSleepCount number of records. try { LOG.info("Reading in " + vertexName @@ -266,16 +273,15 @@ public void map(IntWritable key, IntWritable value, Context context + ", sleepLeft=" + (mapSleepDuration * (mapSleepCount - count))); context.setStatus("Sleeping... (" + - (mapSleepDuration * (mapSleepCount - count)) + ") ms left"); + (mapSleepDuration * (mapSleepCount - count)) + ") ms left"); if ((mapSleepCount - count) > 0) { Thread.sleep(mapSleepDuration); } if (throwError || throwFatal) { throw new IOException("Throwing a simulated error from map"); } - } - catch (InterruptedException ex) { - throw (IOException)new IOException( + } catch (InterruptedException ex) { + throw (IOException) new IOException( "Interrupted while sleeping").initCause(ex); } ++count; @@ -285,14 +291,14 @@ public void map(IntWritable key, IntWritable value, Context context for (int i = 0; i < value.get(); ++i) { LOG.info("Writing in " + vertexName + " taskid " + context.getTaskAttemptID().getTaskID().getId() - + " key " + (k+i) + " value 1"); + + " key " + (k + i) + " value 1"); context.write(new IntWritable(k + i), new IntWritable(1)); } } } public static class ISleepReducer - extends Reducer { + extends Reducer { private long iReduceSleepDuration = 100; private int iReduceSleepCount = 1; private int count = 0; @@ -304,14 +310,14 @@ protected void setup(Context context) this.iReduceSleepCount = conf.getInt(IREDUCE_SLEEP_COUNT, iReduceSleepCount); this.iReduceSleepDuration = iReduceSleepCount == 0 ? 0 : - conf.getLong(IREDUCE_SLEEP_TIME , 100) / iReduceSleepCount; + conf.getLong(IREDUCE_SLEEP_TIME, 100) / iReduceSleepCount; vertexName = conf.get( org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME); } public void reduce(IntWritable key, Iterable values, - Context context) - throws IOException, InterruptedException { + Context context) + throws IOException, InterruptedException { try { LOG.info("Reading in " + vertexName + " taskid " + context.getTaskAttemptID().getTaskID().getId() @@ -325,13 +331,12 @@ public void reduce(IntWritable key, Iterable values, + ", sleepLeft=" + (iReduceSleepDuration * (iReduceSleepCount - count))); context.setStatus("Sleeping... (" + - (iReduceSleepDuration * (iReduceSleepCount - count)) + ") ms left"); + (iReduceSleepDuration * (iReduceSleepCount - count)) + ") ms left"); if ((iReduceSleepCount - count) > 0) { Thread.sleep(iReduceSleepDuration); } - } - catch (InterruptedException ex) { - throw (IOException)new IOException( + } catch (InterruptedException ex) { + throw (IOException) new IOException( "Interrupted while sleeping").initCause(ex); } ++count; @@ -342,7 +347,7 @@ public void reduce(IntWritable key, Iterable values, for (int i = 0; i < value.get(); ++i) { LOG.info("Writing in " + vertexName + " taskid " + context.getTaskAttemptID().getTaskID().getId() - + " key " + (k+i) + " value 1"); + + " key " + (k + i) + " value 1"); context.write(new IntWritable(k + i), new IntWritable(1)); } } @@ -357,19 +362,19 @@ public static class SleepReducer private String vertexName; protected void setup(Context context) - throws IOException, InterruptedException { + throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); this.reduceSleepCount = - conf.getInt(REDUCE_SLEEP_COUNT, reduceSleepCount); + conf.getInt(REDUCE_SLEEP_COUNT, reduceSleepCount); this.reduceSleepDuration = reduceSleepCount == 0 ? 0 : - conf.getLong(REDUCE_SLEEP_TIME , 100) / reduceSleepCount; + conf.getLong(REDUCE_SLEEP_TIME, 100) / reduceSleepCount; vertexName = conf.get( org.apache.tez.mapreduce.hadoop.MRJobConfig.VERTEX_NAME); } public void reduce(IntWritable key, Iterable values, Context context) - throws IOException { + throws IOException { try { LOG.info("Reading in " + vertexName + " taskid " + context.getTaskAttemptID().getTaskID().getId() @@ -386,10 +391,9 @@ public void reduce(IntWritable key, Iterable values, if ((reduceSleepCount - count) > 0) { Thread.sleep(reduceSleepDuration); } - } - catch (InterruptedException ex) { - throw (IOException)new IOException( - "Interrupted while sleeping").initCause(ex); + } catch (InterruptedException ex) { + throw (IOException) new IOException( + "Interrupted while sleeping").initCause(ex); } count++; } @@ -399,18 +403,17 @@ public static void main(String[] args) throws Exception { int res = ToolRunner.run(new Configuration(), new MRRSleepJob(), args); System.exit(res); } - + private Credentials credentials = new Credentials(); public DAG createDAG(Configuration conf, Path stagingDir, - int numMapper, int numReducer, int iReduceStagesCount, - int numIReducer, long mapSleepTime, int mapSleepCount, - long reduceSleepTime, int reduceSleepCount, - long iReduceSleepTime, int iReduceSleepCount, boolean writeSplitsToDFS, - boolean generateSplitsInAM) + int numMapper, int numReducer, int iReduceStagesCount, + int numIReducer, long mapSleepTime, int mapSleepCount, + long reduceSleepTime, int reduceSleepCount, + long iReduceSleepTime, int iReduceSleepCount, boolean writeSplitsToDFS, + boolean generateSplitsInAM) throws IOException, YarnException { - Configuration mapStageConf = new JobConf(conf); mapStageConf.setInt(MRJobConfig.NUM_MAPS, numMapper); mapStageConf.setLong(MAP_SLEEP_TIME, mapSleepTime); @@ -449,9 +452,8 @@ public DAG createDAG(Configuration conf, Path stagingDir, iReduceStageConf.set(MRJobConfig.PARTITIONER_CLASS_ATTR, MRRSleepJobPartitioner.class.getName()); - MRHelpers.translateMRConfToTez(iReduceStageConf, false); - intermediateReduceStageConfs[i-1] = iReduceStageConf; + intermediateReduceStageConfs[i - 1] = iReduceStageConf; } } @@ -496,9 +498,9 @@ public DAG createDAG(Configuration conf, Path stagingDir, DAG dag = DAG.create("MRRSleepJob"); String jarPath = ClassUtil.findContainingJar(getClass()); - if (jarPath == null) { - throw new TezUncheckedException("Could not find any jar containing" - + " MRRSleepJob.class in the classpath"); + if (jarPath == null) { + throw new TezUncheckedException("Could not find any jar containing" + + " MRRSleepJob.class in the classpath"); } FileSystem stagingFs = stagingDir.getFileSystem(conf); @@ -506,7 +508,7 @@ public DAG createDAG(Configuration conf, Path stagingDir, stagingFs.copyFromLocalFile(new Path(jarPath), remoteJarPath); FileStatus jarFileStatus = stagingFs.getFileStatus(remoteJarPath); - TokenCache.obtainTokensForNamenodes(this.credentials, new Path[] { remoteJarPath }, + TokenCache.obtainTokensForNamenodes(this.credentials, new Path[]{remoteJarPath}, mapStageConf); Map commonLocalResources = @@ -521,7 +523,6 @@ public DAG createDAG(Configuration conf, Path stagingDir, List vertices = new ArrayList(); - UserPayload mapUserPayload = TezUtils.createUserPayloadFromConf(mapStageConf); int numTasks = generateSplitsInAM ? -1 : numMapper; @@ -531,7 +532,7 @@ public DAG createDAG(Configuration conf, Path stagingDir, MRHelpers.updateEnvBasedOnMRTaskEnv(mapStageConf, reduceEnv, false); Vertex mapVertex = Vertex.create("map", ProcessorDescriptor.create( - MapProcessor.class.getName()).setUserPayload(mapUserPayload), numTasks, + MapProcessor.class.getName()).setUserPayload(mapUserPayload), numTasks, MRHelpers.getResourceForMRMapper(mapStageConf)); mapVertex.addTaskLocalFiles(commonLocalResources) .addDataSource("MRInput", dataSource) @@ -539,7 +540,7 @@ public DAG createDAG(Configuration conf, Path stagingDir, vertices.add(mapVertex); if (iReduceStagesCount > 0 - && numIReducer > 0) { + && numIReducer > 0) { for (int i = 0; i < iReduceStagesCount; ++i) { Configuration iconf = intermediateReduceStageConfs[i]; @@ -559,7 +560,7 @@ public DAG createDAG(Configuration conf, Path stagingDir, if (numReducer > 0) { UserPayload reducePayload = TezUtils.createUserPayloadFromConf(finalReduceConf); finalReduceVertex = Vertex.create("reduce", ProcessorDescriptor.create( - ReduceProcessor.class.getName()).setUserPayload(reducePayload), numReducer, + ReduceProcessor.class.getName()).setUserPayload(reducePayload), numReducer, MRHelpers.getResourceForMRReducer(finalReduceConf)); finalReduceVertex.addTaskLocalFiles(commonLocalResources) .addDataSink("MROutput", MROutputLegacy.createConfigBuilder( @@ -573,15 +574,14 @@ public DAG createDAG(Configuration conf, Path stagingDir, MROutputLegacy.createConfigBuilder(mapStageConf, NullOutputFormat.class).build()); } - for (int i = 0; i < vertices.size(); ++i) { dag.addVertex(vertices.get(i)); if (i != 0) { Map partitionerConf = Maps.newHashMap(); partitionerConf.put( MRJobConfig.PARTITIONER_CLASS_ATTR, MRRSleepJobPartitioner.class.getName()); - Configuration edgeConfiguration = ((i+1) == vertices.size()) ? - finalReduceConf : intermediateReduceStageConfs[i-1]; + Configuration edgeConfiguration = ((i + 1) == vertices.size()) ? + finalReduceConf : intermediateReduceStageConfs[i - 1]; OrderedPartitionedKVEdgeConfig edgeConf = OrderedPartitionedKVEdgeConfig .newBuilder(IntWritable.class.getName(), IntWritable.class.getName(), HashPartitioner.class.getName(), partitionerConf).configureInput().useLegacyInput() @@ -596,10 +596,10 @@ public DAG createDAG(Configuration conf, Path stagingDir, @VisibleForTesting public Job createJob(int numMapper, int numReducer, int iReduceStagesCount, - int numIReducer, long mapSleepTime, int mapSleepCount, - long reduceSleepTime, int reduceSleepCount, - long iReduceSleepTime, int iReduceSleepCount) - throws IOException { + int numIReducer, long mapSleepTime, int mapSleepCount, + long reduceSleepTime, int reduceSleepCount, + long iReduceSleepTime, int iReduceSleepCount) + throws IOException { Configuration conf = getConf(); conf.setLong(MAP_SLEEP_TIME, mapSleepTime); conf.setLong(REDUCE_SLEEP_TIME, reduceSleepTime); @@ -655,7 +655,7 @@ public Job createJob(int numMapper, int numReducer, int iReduceStagesCount, public int run(String[] args) throws Exception { - if(args.length < 1) { + if (args.length < 1) { System.err.println("MRRSleepJob [-m numMapper] [-r numReducer]" + " [-ir numIntermediateReducer]" + " [-irs numIntermediateReducerStages]" + @@ -671,7 +671,7 @@ public int run(String[] args) throws Exception { int numMapper = 1, numReducer = 1, numIReducer = 1; long mapSleepTime = 100, reduceSleepTime = 100, recSleepTime = 100, - iReduceSleepTime=1; + iReduceSleepTime = 1; int mapSleepCount = 1, reduceSleepCount = 1, iReduceSleepCount = 1; int iReduceStagesCount = 1; boolean writeSplitsToDfs = false; @@ -680,40 +680,30 @@ public int run(String[] args) throws Exception { boolean isSession = false; int numDags = 1; - for(int i=0; i < args.length; i++ ) { - if(args[i].equals("-m")) { + for (int i = 0; i < args.length; i++) { + if (args[i].equals("-m")) { numMapper = Integer.parseInt(args[++i]); - } - else if(args[i].equals("-r")) { + } else if (args[i].equals("-r")) { numReducer = Integer.parseInt(args[++i]); - } - else if(args[i].equals("-ir")) { + } else if (args[i].equals("-ir")) { numIReducer = Integer.parseInt(args[++i]); - } - else if(args[i].equals("-mt")) { + } else if (args[i].equals("-mt")) { mapSleepTime = Long.parseLong(args[++i]); - } - else if(args[i].equals("-rt")) { + } else if (args[i].equals("-rt")) { reduceSleepTime = Long.parseLong(args[++i]); - } - else if(args[i].equals("-irt")) { + } else if (args[i].equals("-irt")) { iReduceSleepTime = Long.parseLong(args[++i]); - } - else if(args[i].equals("-irs")) { + } else if (args[i].equals("-irs")) { iReduceStagesCount = Integer.parseInt(args[++i]); - } - else if (args[i].equals("-recordt")) { + } else if (args[i].equals("-recordt")) { recSleepTime = Long.parseLong(args[++i]); - } - else if (args[i].equals("-generateSplitsInAM")) { + } else if (args[i].equals("-generateSplitsInAM")) { if (splitsOptionFound) { throw new RuntimeException("Cannot use both -generateSplitsInAm and -writeSplitsToDfs together"); } splitsOptionFound = true; generateSplitsInAM = Boolean.parseBoolean(args[++i]); - - } - else if (args[i].equals("-writeSplitsToDfs")) { + } else if (args[i].equals("-writeSplitsToDfs")) { if (splitsOptionFound) { throw new RuntimeException("Cannot use both -generateSplitsInAm and -writeSplitsToDfs together"); } @@ -734,9 +724,9 @@ else if (args[i].equals("-writeSplitsToDfs")) { } // sleep for *SleepTime duration in Task by recSleepTime per record - mapSleepCount = (int)Math.ceil(mapSleepTime / ((double)recSleepTime)); - reduceSleepCount = (int)Math.ceil(reduceSleepTime / ((double)recSleepTime)); - iReduceSleepCount = (int)Math.ceil(iReduceSleepTime / ((double)recSleepTime)); + mapSleepCount = (int) Math.ceil(mapSleepTime / ((double) recSleepTime)); + reduceSleepCount = (int) Math.ceil(reduceSleepTime / ((double) recSleepTime)); + iReduceSleepCount = (int) Math.ceil(iReduceSleepTime / ((double) recSleepTime)); TezConfiguration conf = new TezConfiguration(getConf()); @@ -771,5 +761,4 @@ else if (args[i].equals("-writeSplitsToDfs")) { } return 0; } - } diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MapredWordCount.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MapredWordCount.java index 49b6d797ae..775aae989a 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MapredWordCount.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MapredWordCount.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -64,7 +64,7 @@ public class MapredWordCount extends Configured implements Tool { * (word, 1). */ public static class MapClass extends MapReduceBase - implements Mapper { + implements Mapper { private final static IntWritable one = new IntWritable(1); private Text word = new Text(); @@ -85,7 +85,7 @@ public void map(LongWritable key, Text value, * A reducer class that just emits the sum of the input values. */ public static class Reduce extends MapReduceBase - implements Reducer { + implements Reducer { public void reduce(Text key, Iterator values, OutputCollector output, @@ -125,7 +125,7 @@ public int run(String[] args) throws Exception { conf.setReducerClass(Reduce.class); List other_args = new ArrayList(); - for(int i=0; i < args.length; ++i) { + for (int i = 0; i < args.length; ++i) { try { if ("-m".equals(args[i])) { conf.setNumMapTasks(Integer.parseInt(args[++i])); @@ -138,7 +138,7 @@ public int run(String[] args) throws Exception { LOG.error("Integer expected instead of " + args[i]); return printUsage(); } catch (ArrayIndexOutOfBoundsException except) { - LOG.error("Required parameter missing from " + args[i-1]); + LOG.error("Required parameter missing from " + args[i - 1]); return printUsage(); } } @@ -157,8 +157,7 @@ public int run(String[] args) throws Exception { public static void main(String[] args) throws Exception { int res = ToolRunner.run(getTezDecoratedConfiguration(), - new MapredWordCount(), args); + new MapredWordCount(), args); System.exit(res); } - } diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MultipleCommitsExample.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MultipleCommitsExample.java index 5c93e87f74..5243f29b42 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MultipleCommitsExample.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MultipleCommitsExample.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -103,39 +103,39 @@ public void initialize() throws Exception { super.initialize(); config = MultipleOutputProcessorConfig.fromUserPayload(getContext().getUserPayload()); } - + @Override public void run() throws Exception { - for (int i=0;i < config.outputNum;++i) { + for (int i = 0; i < config.outputNum; ++i) { KeyValueWriter writer = (KeyValueWriter) - getOutputs().get(config.outputNamePrefix+"_" + i).getWriter(); + getOutputs().get(config.outputNamePrefix + "_" + i).getWriter(); writer.write(NullWritable.get(), new Text("dummy")); } - for (int i=0;i < config.sharedOutputNum; ++i) { + for (int i = 0; i < config.sharedOutputNum; ++i) { KeyValueWriter writer = (KeyValueWriter) - getOutputs().get(config.sharedOutputNamePrefix +"_" + i).getWriter(); + getOutputs().get(config.sharedOutputNamePrefix + "_" + i).getWriter(); writer.write(NullWritable.get(), new Text("dummy")); } } - + public static class MultipleOutputProcessorConfig implements Writable { - + String outputNamePrefix; int outputNum; String sharedOutputNamePrefix = null; int sharedOutputNum; - public MultipleOutputProcessorConfig(){ - + public MultipleOutputProcessorConfig() { + } - + public MultipleOutputProcessorConfig(String outputNamePrefix, int outputNum) { this.outputNamePrefix = outputNamePrefix; this.outputNum = outputNum; } public MultipleOutputProcessorConfig(String outputNamePrefix, int outputNum, - String sharedOutputNamePrefix, int sharedOutputNum) { + String sharedOutputNamePrefix, int sharedOutputNum) { this.outputNamePrefix = outputNamePrefix; this.outputNum = outputNum; this.sharedOutputNamePrefix = sharedOutputNamePrefix; @@ -170,7 +170,7 @@ public void readFields(DataInput in) throws IOException { sharedOutputNum = in.readInt(); } } - + public UserPayload toUserPayload() throws IOException { NonSyncByteArrayOutputStream out = new NonSyncByteArrayOutputStream(); this.write(new NonSyncDataOutputStream(out)); @@ -189,7 +189,7 @@ public static MultipleOutputProcessorConfig fromUserPayload(UserPayload payload) @Override protected int runJob(String[] args, TezConfiguration tezConf, - TezClient tezClient) throws Exception { + TezClient tezClient) throws Exception { boolean commitOnVertexSuccess = args.length == 5 && args[4].equals(CommitOnVertexSuccessOption) ? true : false; DAG dag = createDAG(tezConf, args[0], Integer.parseInt(args[1]), @@ -201,37 +201,37 @@ protected int runJob(String[] args, TezConfiguration tezConf, return runDag(dag, false, LOG); } - private DAG createDAG(TezConfiguration tezConf, - String v1OutputPathPrefix, int v1OutputNum, String v2OutputPathPrefix, int v2OutputNum, - String uv12OutputPathPrefix, int uv12OutputNum, - String v3OutputPathPrefix, int v3OutputNum, boolean commitOnVertexSuccess) throws IOException { + private DAG createDAG(TezConfiguration tezConf, + String v1OutputPathPrefix, int v1OutputNum, String v2OutputPathPrefix, int v2OutputNum, + String uv12OutputPathPrefix, int uv12OutputNum, + String v3OutputPathPrefix, int v3OutputNum, boolean commitOnVertexSuccess) throws IOException { DAG dag = DAG.create("multipleCommitsDAG"); dag.setConf(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, !commitOnVertexSuccess + ""); Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create(MultipleOutputProcessor.class.getName()) .setUserPayload( new MultipleOutputProcessor.MultipleOutputProcessorConfig( V1OutputNamePrefix, v1OutputNum, UV12OutputNamePrefix, uv12OutputNum) - .toUserPayload()), 2); + .toUserPayload()), 2); Vertex v2 = Vertex.create("v2", ProcessorDescriptor.create(MultipleOutputProcessor.class.getName()) .setUserPayload( new MultipleOutputProcessor.MultipleOutputProcessorConfig( V2OutputNamePrefix, v2OutputNum, UV12OutputNamePrefix, uv12OutputNum) - .toUserPayload()), 2); + .toUserPayload()), 2); // add data sinks for v1 - for (int i=0;i=3 && otherArgs.length <=4) ? 0 : 2; + return (otherArgs.length >= 3 && otherArgs.length <= 4) ? 0 : 2; } private DAG createDAG(TezConfiguration conf, int numTasks, int maxSleepTimeMillis, @@ -156,7 +156,7 @@ private UserPayload createUserPayload(TezConfiguration conf, int maxSleepTimeMil FSDataOutputStream dataOut = fs.create(resourcePath, true); dataOut.write(diskPayload); dataOut.close(); - fs.setReplication(resourcePath, (short)10); + fs.setReplication(resourcePath, (short) 10); FileStatus fileStatus = fs.getFileStatus(resourcePath); if (mode.equals(VIA_HDFS_DIST_CACHE)) { diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomTextWriter.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomTextWriter.java index 55404ba1bd..53cb9bbe48 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomTextWriter.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomTextWriter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -69,184 +69,184 @@ * 1099511627776 * * - * + * * Equivalently, {@link RandomTextWriter} also supports all the above options * and ones supported by {@link Tool} via the command-line. - * + * * To run: bin/hadoop jar hadoop-${version}-examples.jar randomtextwriter * [-outFormat output format class] output */ public class RandomTextWriter extends Configured implements Tool { - public static final String TOTAL_BYTES = - "mapreduce.randomtextwriter.totalbytes"; - public static final String BYTES_PER_MAP = - "mapreduce.randomtextwriter.bytespermap"; - public static final String MAPS_PER_HOST = - "mapreduce.randomtextwriter.mapsperhost"; + public static final String TOTAL_BYTES = + "mapreduce.randomtextwriter.totalbytes"; + public static final String BYTES_PER_MAP = + "mapreduce.randomtextwriter.bytespermap"; + public static final String MAPS_PER_HOST = + "mapreduce.randomtextwriter.mapsperhost"; public static final String MAX_VALUE = "mapreduce.randomtextwriter.maxwordsvalue"; public static final String MIN_VALUE = "mapreduce.randomtextwriter.minwordsvalue"; public static final String MIN_KEY = "mapreduce.randomtextwriter.minwordskey"; public static final String MAX_KEY = "mapreduce.randomtextwriter.maxwordskey"; - + static int printUsage() { System.out.println("randomtextwriter " + - "[-outFormat ] " + - ""); + "[-outFormat ] " + + ""); ToolRunner.printGenericCommandUsage(System.out); return 2; } - + /** * User counters */ - static enum Counters { RECORDS_WRITTEN, BYTES_WRITTEN } + static enum Counters {RECORDS_WRITTEN, BYTES_WRITTEN} static class RandomTextMapper extends Mapper { - + private long numBytesToWrite; private int minWordsInKey; private int wordsInKeyRange; private int minWordsInValue; private int wordsInValueRange; private Random random = new Random(); - + /** * Save the configuration value that we need to write the data. */ public void setup(Context context) { Configuration conf = context.getConfiguration(); numBytesToWrite = conf.getLong(BYTES_PER_MAP, - 1*1024*1024*1024); + 1 * 1024 * 1024 * 1024); minWordsInKey = conf.getInt(MIN_KEY, 5); wordsInKeyRange = (conf.getInt(MAX_KEY, 10) - minWordsInKey); minWordsInValue = conf.getInt(MIN_VALUE, 10); wordsInValueRange = (conf.getInt(MAX_VALUE, 100) - minWordsInValue); } - + /** * Given an output filename, write a bunch of random records to it. */ public void map(Text key, Text value, - Context context) throws IOException,InterruptedException { + Context context) throws IOException, InterruptedException { int itemCount = 0; while (numBytesToWrite > 0) { // Generate the key/value - int noWordsKey = minWordsInKey + - (wordsInKeyRange != 0 ? random.nextInt(wordsInKeyRange) : 0); - int noWordsValue = minWordsInValue + - (wordsInValueRange != 0 ? random.nextInt(wordsInValueRange) : 0); + int noWordsKey = minWordsInKey + + (wordsInKeyRange != 0 ? random.nextInt(wordsInKeyRange) : 0); + int noWordsValue = minWordsInValue + + (wordsInValueRange != 0 ? random.nextInt(wordsInValueRange) : 0); Text keyWords = generateSentence(noWordsKey); Text valueWords = generateSentence(noWordsValue); - + // Write the sentence context.write(keyWords, valueWords); - + numBytesToWrite -= (keyWords.getLength() + valueWords.getLength()); - + // Update counters, progress etc. context.getCounter(Counters.BYTES_WRITTEN).increment( - keyWords.getLength() + valueWords.getLength()); + keyWords.getLength() + valueWords.getLength()); context.getCounter(Counters.RECORDS_WRITTEN).increment(1); if (++itemCount % 200 == 0) { - context.setStatus("wrote record " + itemCount + ". " + - numBytesToWrite + " bytes left."); + context.setStatus("wrote record " + itemCount + ". " + + numBytesToWrite + " bytes left."); } } context.setStatus("done with " + itemCount + " records."); } - + private Text generateSentence(int noWords) { StringBuffer sentence = new StringBuffer(); String space = " "; - for (int i=0; i < noWords; ++i) { + for (int i = 0; i < noWords; ++i) { sentence.append(words[random.nextInt(words.length)]); sentence.append(space); } return new Text(sentence.toString()); } } - + /** * This is the main routine for launching a distributed random write job. * It runs 10 maps/node and each node writes 1 gig of data to a DFS file. * The reduce doesn't do anything. - * - * @throws IOException + * + * @throws IOException */ @SuppressWarnings("deprecation") - public int run(String[] args) throws Exception { + public int run(String[] args) throws Exception { if (args.length == 0) { - return printUsage(); + return printUsage(); } - + Configuration conf = getConf(); JobClient client = new JobClient(conf); ClusterStatus cluster = client.getClusterStatus(); int numMapsPerHost = conf.getInt(MAPS_PER_HOST, 10); long numBytesToWritePerMap = conf.getLong(BYTES_PER_MAP, - 1*1024*1024*1024); + 1 * 1024 * 1024 * 1024); if (numBytesToWritePerMap == 0) { - System.err.println("Cannot have " + BYTES_PER_MAP +" set to 0"); + System.err.println("Cannot have " + BYTES_PER_MAP + " set to 0"); return -2; } - long totalBytesToWrite = conf.getLong(TOTAL_BYTES, - numMapsPerHost*numBytesToWritePerMap*cluster.getTaskTrackers()); + long totalBytesToWrite = conf.getLong(TOTAL_BYTES, + numMapsPerHost * numBytesToWritePerMap * cluster.getTaskTrackers()); int numMaps = (int) (totalBytesToWrite / numBytesToWritePerMap); if (numMaps == 0 && totalBytesToWrite > 0) { numMaps = 1; conf.setLong(BYTES_PER_MAP, totalBytesToWrite); } conf.setInt(MRJobConfig.NUM_MAPS, numMaps); - + Job job = new Job(conf); - + job.setJarByClass(RandomTextWriter.class); job.setJobName("random-text-writer"); - + job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); - + job.setInputFormatClass(RandomWriter.RandomInputFormat.class); - job.setMapperClass(RandomTextMapper.class); - - Class outputFormatClass = - SequenceFileOutputFormat.class; + job.setMapperClass(RandomTextMapper.class); + + Class outputFormatClass = + SequenceFileOutputFormat.class; List otherArgs = new ArrayList(); - for(int i=0; i < args.length; ++i) { + for (int i = 0; i < args.length; ++i) { try { if ("-outFormat".equals(args[i])) { - outputFormatClass = - Class.forName(args[++i]).asSubclass(OutputFormat.class); + outputFormatClass = + Class.forName(args[++i]).asSubclass(OutputFormat.class); } else { otherArgs.add(args[i]); } } catch (ArrayIndexOutOfBoundsException except) { System.out.println("ERROR: Required parameter missing from " + - args[i-1]); + args[i - 1]); return printUsage(); // exits } } job.setOutputFormatClass(outputFormatClass); FileOutputFormat.setOutputPath(job, new Path(otherArgs.get(0))); - + System.out.println("Running " + numMaps + " maps."); - + // reducer NONE job.setNumReduceTasks(0); - + Date startTime = new Date(); System.out.println("Job started: " + startTime); int ret = job.waitForCompletion(true) ? 0 : 1; Date endTime = new Date(); System.out.println("Job ended: " + endTime); - System.out.println("The job took " + - (endTime.getTime() - startTime.getTime()) /1000 + - " seconds."); - + System.out.println("The job took " + + (endTime.getTime() - startTime.getTime()) / 1000 + + " seconds."); + return ret; } - + public static void main(String[] args) throws Exception { int res = ToolRunner.run(getTezDecoratedConfiguration(), new RandomTextWriter(), args); System.exit(res); @@ -256,505 +256,505 @@ public static void main(String[] args) throws Exception { * A random list of 100 words from /usr/share/dict/words */ private static String[] words = { - "diurnalness", "Homoiousian", - "spiranthic", "tetragynian", - "silverhead", "ungreat", - "lithograph", "exploiter", - "physiologian", "by", - "hellbender", "Filipendula", - "undeterring", "antiscolic", - "pentagamist", "hypoid", - "cacuminal", "sertularian", - "schoolmasterism", "nonuple", - "gallybeggar", "phytonic", - "swearingly", "nebular", - "Confervales", "thermochemically", - "characinoid", "cocksuredom", - "fallacious", "feasibleness", - "debromination", "playfellowship", - "tramplike", "testa", - "participatingly", "unaccessible", - "bromate", "experientialist", - "roughcast", "docimastical", - "choralcelo", "blightbird", - "peptonate", "sombreroed", - "unschematized", "antiabolitionist", - "besagne", "mastication", - "bromic", "sviatonosite", - "cattimandoo", "metaphrastical", - "endotheliomyoma", "hysterolysis", - "unfulminated", "Hester", - "oblongly", "blurredness", - "authorling", "chasmy", - "Scorpaenidae", "toxihaemia", - "Dictograph", "Quakerishly", - "deaf", "timbermonger", - "strammel", "Thraupidae", - "seditious", "plerome", - "Arneb", "eristically", - "serpentinic", "glaumrie", - "socioromantic", "apocalypst", - "tartrous", "Bassaris", - "angiolymphoma", "horsefly", - "kenno", "astronomize", - "euphemious", "arsenide", - "untongued", "parabolicness", - "uvanite", "helpless", - "gemmeous", "stormy", - "templar", "erythrodextrin", - "comism", "interfraternal", - "preparative", "parastas", - "frontoorbital", "Ophiosaurus", - "diopside", "serosanguineous", - "ununiformly", "karyological", - "collegian", "allotropic", - "depravity", "amylogenesis", - "reformatory", "epidymides", - "pleurotropous", "trillium", - "dastardliness", "coadvice", - "embryotic", "benthonic", - "pomiferous", "figureheadship", - "Megaluridae", "Harpa", - "frenal", "commotion", - "abthainry", "cobeliever", - "manilla", "spiciferous", - "nativeness", "obispo", - "monilioid", "biopsic", - "valvula", "enterostomy", - "planosubulate", "pterostigma", - "lifter", "triradiated", - "venialness", "tum", - "archistome", "tautness", - "unswanlike", "antivenin", - "Lentibulariaceae", "Triphora", - "angiopathy", "anta", - "Dawsonia", "becomma", - "Yannigan", "winterproof", - "antalgol", "harr", - "underogating", "ineunt", - "cornberry", "flippantness", - "scyphostoma", "approbation", - "Ghent", "Macraucheniidae", - "scabbiness", "unanatomized", - "photoelasticity", "eurythermal", - "enation", "prepavement", - "flushgate", "subsequentially", - "Edo", "antihero", - "Isokontae", "unforkedness", - "porriginous", "daytime", - "nonexecutive", "trisilicic", - "morphiomania", "paranephros", - "botchedly", "impugnation", - "Dodecatheon", "obolus", - "unburnt", "provedore", - "Aktistetae", "superindifference", - "Alethea", "Joachimite", - "cyanophilous", "chorograph", - "brooky", "figured", - "periclitation", "quintette", - "hondo", "ornithodelphous", - "unefficient", "pondside", - "bogydom", "laurinoxylon", - "Shiah", "unharmed", - "cartful", "noncrystallized", - "abusiveness", "cromlech", - "japanned", "rizzomed", - "underskin", "adscendent", - "allectory", "gelatinousness", - "volcano", "uncompromisingly", - "cubit", "idiotize", - "unfurbelowed", "undinted", - "magnetooptics", "Savitar", - "diwata", "ramosopalmate", - "Pishquow", "tomorn", - "apopenptic", "Haversian", - "Hysterocarpus", "ten", - "outhue", "Bertat", - "mechanist", "asparaginic", - "velaric", "tonsure", - "bubble", "Pyrales", - "regardful", "glyphography", - "calabazilla", "shellworker", - "stradametrical", "havoc", - "theologicopolitical", "sawdust", - "diatomaceous", "jajman", - "temporomastoid", "Serrifera", - "Ochnaceae", "aspersor", - "trailmaking", "Bishareen", - "digitule", "octogynous", - "epididymitis", "smokefarthings", - "bacillite", "overcrown", - "mangonism", "sirrah", - "undecorated", "psychofugal", - "bismuthiferous", "rechar", - "Lemuridae", "frameable", - "thiodiazole", "Scanic", - "sportswomanship", "interruptedness", - "admissory", "osteopaedion", - "tingly", "tomorrowness", - "ethnocracy", "trabecular", - "vitally", "fossilism", - "adz", "metopon", - "prefatorial", "expiscate", - "diathermacy", "chronist", - "nigh", "generalizable", - "hysterogen", "aurothiosulphuric", - "whitlowwort", "downthrust", - "Protestantize", "monander", - "Itea", "chronographic", - "silicize", "Dunlop", - "eer", "componental", - "spot", "pamphlet", - "antineuritic", "paradisean", - "interruptor", "debellator", - "overcultured", "Florissant", - "hyocholic", "pneumatotherapy", - "tailoress", "rave", - "unpeople", "Sebastian", - "thermanesthesia", "Coniferae", - "swacking", "posterishness", - "ethmopalatal", "whittle", - "analgize", "scabbardless", - "naught", "symbiogenetically", - "trip", "parodist", - "columniform", "trunnel", - "yawler", "goodwill", - "pseudohalogen", "swangy", - "cervisial", "mediateness", - "genii", "imprescribable", - "pony", "consumptional", - "carposporangial", "poleax", - "bestill", "subfebrile", - "sapphiric", "arrowworm", - "qualminess", "ultraobscure", - "thorite", "Fouquieria", - "Bermudian", "prescriber", - "elemicin", "warlike", - "semiangle", "rotular", - "misthread", "returnability", - "seraphism", "precostal", - "quarried", "Babylonism", - "sangaree", "seelful", - "placatory", "pachydermous", - "bozal", "galbulus", - "spermaphyte", "cumbrousness", - "pope", "signifier", - "Endomycetaceae", "shallowish", - "sequacity", "periarthritis", - "bathysphere", "pentosuria", - "Dadaism", "spookdom", - "Consolamentum", "afterpressure", - "mutter", "louse", - "ovoviviparous", "corbel", - "metastoma", "biventer", - "Hydrangea", "hogmace", - "seizing", "nonsuppressed", - "oratorize", "uncarefully", - "benzothiofuran", "penult", - "balanocele", "macropterous", - "dishpan", "marten", - "absvolt", "jirble", - "parmelioid", "airfreighter", - "acocotl", "archesporial", - "hypoplastral", "preoral", - "quailberry", "cinque", - "terrestrially", "stroking", - "limpet", "moodishness", - "canicule", "archididascalian", - "pompiloid", "overstaid", - "introducer", "Italical", - "Christianopaganism", "prescriptible", - "subofficer", "danseuse", - "cloy", "saguran", - "frictionlessly", "deindividualization", - "Bulanda", "ventricous", - "subfoliar", "basto", - "scapuloradial", "suspend", - "stiffish", "Sphenodontidae", - "eternal", "verbid", - "mammonish", "upcushion", - "barkometer", "concretion", - "preagitate", "incomprehensible", - "tristich", "visceral", - "hemimelus", "patroller", - "stentorophonic", "pinulus", - "kerykeion", "brutism", - "monstership", "merciful", - "overinstruct", "defensibly", - "bettermost", "splenauxe", - "Mormyrus", "unreprimanded", - "taver", "ell", - "proacquittal", "infestation", - "overwoven", "Lincolnlike", - "chacona", "Tamil", - "classificational", "lebensraum", - "reeveland", "intuition", - "Whilkut", "focaloid", - "Eleusinian", "micromembrane", - "byroad", "nonrepetition", - "bacterioblast", "brag", - "ribaldrous", "phytoma", - "counteralliance", "pelvimetry", - "pelf", "relaster", - "thermoresistant", "aneurism", - "molossic", "euphonym", - "upswell", "ladhood", - "phallaceous", "inertly", - "gunshop", "stereotypography", - "laryngic", "refasten", - "twinling", "oflete", - "hepatorrhaphy", "electrotechnics", - "cockal", "guitarist", - "topsail", "Cimmerianism", - "larklike", "Llandovery", - "pyrocatechol", "immatchable", - "chooser", "metrocratic", - "craglike", "quadrennial", - "nonpoisonous", "undercolored", - "knob", "ultratense", - "balladmonger", "slait", - "sialadenitis", "bucketer", - "magnificently", "unstipulated", - "unscourged", "unsupercilious", - "packsack", "pansophism", - "soorkee", "percent", - "subirrigate", "champer", - "metapolitics", "spherulitic", - "involatile", "metaphonical", - "stachyuraceous", "speckedness", - "bespin", "proboscidiform", - "gul", "squit", - "yeelaman", "peristeropode", - "opacousness", "shibuichi", - "retinize", "yote", - "misexposition", "devilwise", - "pumpkinification", "vinny", - "bonze", "glossing", - "decardinalize", "transcortical", - "serphoid", "deepmost", - "guanajuatite", "wemless", - "arval", "lammy", - "Effie", "Saponaria", - "tetrahedral", "prolificy", - "excerpt", "dunkadoo", - "Spencerism", "insatiately", - "Gilaki", "oratorship", - "arduousness", "unbashfulness", - "Pithecolobium", "unisexuality", - "veterinarian", "detractive", - "liquidity", "acidophile", - "proauction", "sural", - "totaquina", "Vichyite", - "uninhabitedness", "allegedly", - "Gothish", "manny", - "Inger", "flutist", - "ticktick", "Ludgatian", - "homotransplant", "orthopedical", - "diminutively", "monogoneutic", - "Kenipsim", "sarcologist", - "drome", "stronghearted", - "Fameuse", "Swaziland", - "alen", "chilblain", - "beatable", "agglomeratic", - "constitutor", "tendomucoid", - "porencephalous", "arteriasis", - "boser", "tantivy", - "rede", "lineamental", - "uncontradictableness", "homeotypical", - "masa", "folious", - "dosseret", "neurodegenerative", - "subtransverse", "Chiasmodontidae", - "palaeotheriodont", "unstressedly", - "chalcites", "piquantness", - "lampyrine", "Aplacentalia", - "projecting", "elastivity", - "isopelletierin", "bladderwort", - "strander", "almud", - "iniquitously", "theologal", - "bugre", "chargeably", - "imperceptivity", "meriquinoidal", - "mesophyte", "divinator", - "perfunctory", "counterappellant", - "synovial", "charioteer", - "crystallographical", "comprovincial", - "infrastapedial", "pleasurehood", - "inventurous", "ultrasystematic", - "subangulated", "supraoesophageal", - "Vaishnavism", "transude", - "chrysochrous", "ungrave", - "reconciliable", "uninterpleaded", - "erlking", "wherefrom", - "aprosopia", "antiadiaphorist", - "metoxazine", "incalculable", - "umbellic", "predebit", - "foursquare", "unimmortal", - "nonmanufacture", "slangy", - "predisputant", "familist", - "preaffiliate", "friarhood", - "corelysis", "zoonitic", - "halloo", "paunchy", - "neuromimesis", "aconitine", - "hackneyed", "unfeeble", - "cubby", "autoschediastical", - "naprapath", "lyrebird", - "inexistency", "leucophoenicite", - "ferrogoslarite", "reperuse", - "uncombable", "tambo", - "propodiale", "diplomatize", - "Russifier", "clanned", - "corona", "michigan", - "nonutilitarian", "transcorporeal", - "bought", "Cercosporella", - "stapedius", "glandularly", - "pictorially", "weism", - "disilane", "rainproof", - "Caphtor", "scrubbed", - "oinomancy", "pseudoxanthine", - "nonlustrous", "redesertion", - "Oryzorictinae", "gala", - "Mycogone", "reappreciate", - "cyanoguanidine", "seeingness", - "breadwinner", "noreast", - "furacious", "epauliere", - "omniscribent", "Passiflorales", - "uninductive", "inductivity", - "Orbitolina", "Semecarpus", - "migrainoid", "steprelationship", - "phlogisticate", "mesymnion", - "sloped", "edificator", - "beneficent", "culm", - "paleornithology", "unurban", - "throbless", "amplexifoliate", - "sesquiquintile", "sapience", - "astucious", "dithery", - "boor", "ambitus", - "scotching", "uloid", - "uncompromisingness", "hoove", - "waird", "marshiness", - "Jerusalem", "mericarp", - "unevoked", "benzoperoxide", - "outguess", "pyxie", - "hymnic", "euphemize", - "mendacity", "erythremia", - "rosaniline", "unchatteled", - "lienteria", "Bushongo", - "dialoguer", "unrepealably", - "rivethead", "antideflation", - "vinegarish", "manganosiderite", - "doubtingness", "ovopyriform", - "Cephalodiscus", "Muscicapa", - "Animalivora", "angina", - "planispheric", "ipomoein", - "cuproiodargyrite", "sandbox", - "scrat", "Munnopsidae", - "shola", "pentafid", - "overstudiousness", "times", - "nonprofession", "appetible", - "valvulotomy", "goladar", - "uniarticular", "oxyterpene", - "unlapsing", "omega", - "trophonema", "seminonflammable", - "circumzenithal", "starer", - "depthwise", "liberatress", - "unleavened", "unrevolting", - "groundneedle", "topline", - "wandoo", "umangite", - "ordinant", "unachievable", - "oversand", "snare", - "avengeful", "unexplicit", - "mustafina", "sonable", - "rehabilitative", "eulogization", - "papery", "technopsychology", - "impressor", "cresylite", - "entame", "transudatory", - "scotale", "pachydermatoid", - "imaginary", "yeat", - "slipped", "stewardship", - "adatom", "cockstone", - "skyshine", "heavenful", - "comparability", "exprobratory", - "dermorhynchous", "parquet", - "cretaceous", "vesperal", - "raphis", "undangered", - "Glecoma", "engrain", - "counteractively", "Zuludom", - "orchiocatabasis", "Auriculariales", - "warriorwise", "extraorganismal", - "overbuilt", "alveolite", - "tetchy", "terrificness", - "widdle", "unpremonished", - "rebilling", "sequestrum", - "equiconvex", "heliocentricism", - "catabaptist", "okonite", - "propheticism", "helminthagogic", - "calycular", "giantly", - "wingable", "golem", - "unprovided", "commandingness", - "greave", "haply", - "doina", "depressingly", - "subdentate", "impairment", - "decidable", "neurotrophic", - "unpredict", "bicorporeal", - "pendulant", "flatman", - "intrabred", "toplike", - "Prosobranchiata", "farrantly", - "toxoplasmosis", "gorilloid", - "dipsomaniacal", "aquiline", - "atlantite", "ascitic", - "perculsive", "prospectiveness", - "saponaceous", "centrifugalization", - "dinical", "infravaginal", - "beadroll", "affaite", - "Helvidian", "tickleproof", - "abstractionism", "enhedge", - "outwealth", "overcontribute", - "coldfinch", "gymnastic", - "Pincian", "Munychian", - "codisjunct", "quad", - "coracomandibular", "phoenicochroite", - "amender", "selectivity", - "putative", "semantician", - "lophotrichic", "Spatangoidea", - "saccharogenic", "inferent", - "Triconodonta", "arrendation", - "sheepskin", "taurocolla", - "bunghole", "Machiavel", - "triakistetrahedral", "dehairer", - "prezygapophysial", "cylindric", - "pneumonalgia", "sleigher", - "emir", "Socraticism", - "licitness", "massedly", - "instructiveness", "sturdied", - "redecrease", "starosta", - "evictor", "orgiastic", - "squdge", "meloplasty", - "Tsonecan", "repealableness", - "swoony", "myesthesia", - "molecule", "autobiographist", - "reciprocation", "refective", - "unobservantness", "tricae", - "ungouged", "floatability", - "Mesua", "fetlocked", - "chordacentrum", "sedentariness", - "various", "laubanite", - "nectopod", "zenick", - "sequentially", "analgic", - "biodynamics", "posttraumatic", - "nummi", "pyroacetic", - "bot", "redescend", - "dispermy", "undiffusive", - "circular", "trillion", - "Uraniidae", "ploration", - "discipular", "potentness", - "sud", "Hu", - "Eryon", "plugger", - "subdrainage", "jharal", - "abscission", "supermarket", - "countergabion", "glacierist", - "lithotresis", "minniebush", - "zanyism", "eucalypteol", - "sterilely", "unrealize", - "unpatched", "hypochondriacism", - "critically", "cheesecutter", - }; + "diurnalness", "Homoiousian", + "spiranthic", "tetragynian", + "silverhead", "ungreat", + "lithograph", "exploiter", + "physiologian", "by", + "hellbender", "Filipendula", + "undeterring", "antiscolic", + "pentagamist", "hypoid", + "cacuminal", "sertularian", + "schoolmasterism", "nonuple", + "gallybeggar", "phytonic", + "swearingly", "nebular", + "Confervales", "thermochemically", + "characinoid", "cocksuredom", + "fallacious", "feasibleness", + "debromination", "playfellowship", + "tramplike", "testa", + "participatingly", "unaccessible", + "bromate", "experientialist", + "roughcast", "docimastical", + "choralcelo", "blightbird", + "peptonate", "sombreroed", + "unschematized", "antiabolitionist", + "besagne", "mastication", + "bromic", "sviatonosite", + "cattimandoo", "metaphrastical", + "endotheliomyoma", "hysterolysis", + "unfulminated", "Hester", + "oblongly", "blurredness", + "authorling", "chasmy", + "Scorpaenidae", "toxihaemia", + "Dictograph", "Quakerishly", + "deaf", "timbermonger", + "strammel", "Thraupidae", + "seditious", "plerome", + "Arneb", "eristically", + "serpentinic", "glaumrie", + "socioromantic", "apocalypst", + "tartrous", "Bassaris", + "angiolymphoma", "horsefly", + "kenno", "astronomize", + "euphemious", "arsenide", + "untongued", "parabolicness", + "uvanite", "helpless", + "gemmeous", "stormy", + "templar", "erythrodextrin", + "comism", "interfraternal", + "preparative", "parastas", + "frontoorbital", "Ophiosaurus", + "diopside", "serosanguineous", + "ununiformly", "karyological", + "collegian", "allotropic", + "depravity", "amylogenesis", + "reformatory", "epidymides", + "pleurotropous", "trillium", + "dastardliness", "coadvice", + "embryotic", "benthonic", + "pomiferous", "figureheadship", + "Megaluridae", "Harpa", + "frenal", "commotion", + "abthainry", "cobeliever", + "manilla", "spiciferous", + "nativeness", "obispo", + "monilioid", "biopsic", + "valvula", "enterostomy", + "planosubulate", "pterostigma", + "lifter", "triradiated", + "venialness", "tum", + "archistome", "tautness", + "unswanlike", "antivenin", + "Lentibulariaceae", "Triphora", + "angiopathy", "anta", + "Dawsonia", "becomma", + "Yannigan", "winterproof", + "antalgol", "harr", + "underogating", "ineunt", + "cornberry", "flippantness", + "scyphostoma", "approbation", + "Ghent", "Macraucheniidae", + "scabbiness", "unanatomized", + "photoelasticity", "eurythermal", + "enation", "prepavement", + "flushgate", "subsequentially", + "Edo", "antihero", + "Isokontae", "unforkedness", + "porriginous", "daytime", + "nonexecutive", "trisilicic", + "morphiomania", "paranephros", + "botchedly", "impugnation", + "Dodecatheon", "obolus", + "unburnt", "provedore", + "Aktistetae", "superindifference", + "Alethea", "Joachimite", + "cyanophilous", "chorograph", + "brooky", "figured", + "periclitation", "quintette", + "hondo", "ornithodelphous", + "unefficient", "pondside", + "bogydom", "laurinoxylon", + "Shiah", "unharmed", + "cartful", "noncrystallized", + "abusiveness", "cromlech", + "japanned", "rizzomed", + "underskin", "adscendent", + "allectory", "gelatinousness", + "volcano", "uncompromisingly", + "cubit", "idiotize", + "unfurbelowed", "undinted", + "magnetooptics", "Savitar", + "diwata", "ramosopalmate", + "Pishquow", "tomorn", + "apopenptic", "Haversian", + "Hysterocarpus", "ten", + "outhue", "Bertat", + "mechanist", "asparaginic", + "velaric", "tonsure", + "bubble", "Pyrales", + "regardful", "glyphography", + "calabazilla", "shellworker", + "stradametrical", "havoc", + "theologicopolitical", "sawdust", + "diatomaceous", "jajman", + "temporomastoid", "Serrifera", + "Ochnaceae", "aspersor", + "trailmaking", "Bishareen", + "digitule", "octogynous", + "epididymitis", "smokefarthings", + "bacillite", "overcrown", + "mangonism", "sirrah", + "undecorated", "psychofugal", + "bismuthiferous", "rechar", + "Lemuridae", "frameable", + "thiodiazole", "Scanic", + "sportswomanship", "interruptedness", + "admissory", "osteopaedion", + "tingly", "tomorrowness", + "ethnocracy", "trabecular", + "vitally", "fossilism", + "adz", "metopon", + "prefatorial", "expiscate", + "diathermacy", "chronist", + "nigh", "generalizable", + "hysterogen", "aurothiosulphuric", + "whitlowwort", "downthrust", + "Protestantize", "monander", + "Itea", "chronographic", + "silicize", "Dunlop", + "eer", "componental", + "spot", "pamphlet", + "antineuritic", "paradisean", + "interruptor", "debellator", + "overcultured", "Florissant", + "hyocholic", "pneumatotherapy", + "tailoress", "rave", + "unpeople", "Sebastian", + "thermanesthesia", "Coniferae", + "swacking", "posterishness", + "ethmopalatal", "whittle", + "analgize", "scabbardless", + "naught", "symbiogenetically", + "trip", "parodist", + "columniform", "trunnel", + "yawler", "goodwill", + "pseudohalogen", "swangy", + "cervisial", "mediateness", + "genii", "imprescribable", + "pony", "consumptional", + "carposporangial", "poleax", + "bestill", "subfebrile", + "sapphiric", "arrowworm", + "qualminess", "ultraobscure", + "thorite", "Fouquieria", + "Bermudian", "prescriber", + "elemicin", "warlike", + "semiangle", "rotular", + "misthread", "returnability", + "seraphism", "precostal", + "quarried", "Babylonism", + "sangaree", "seelful", + "placatory", "pachydermous", + "bozal", "galbulus", + "spermaphyte", "cumbrousness", + "pope", "signifier", + "Endomycetaceae", "shallowish", + "sequacity", "periarthritis", + "bathysphere", "pentosuria", + "Dadaism", "spookdom", + "Consolamentum", "afterpressure", + "mutter", "louse", + "ovoviviparous", "corbel", + "metastoma", "biventer", + "Hydrangea", "hogmace", + "seizing", "nonsuppressed", + "oratorize", "uncarefully", + "benzothiofuran", "penult", + "balanocele", "macropterous", + "dishpan", "marten", + "absvolt", "jirble", + "parmelioid", "airfreighter", + "acocotl", "archesporial", + "hypoplastral", "preoral", + "quailberry", "cinque", + "terrestrially", "stroking", + "limpet", "moodishness", + "canicule", "archididascalian", + "pompiloid", "overstaid", + "introducer", "Italical", + "Christianopaganism", "prescriptible", + "subofficer", "danseuse", + "cloy", "saguran", + "frictionlessly", "deindividualization", + "Bulanda", "ventricous", + "subfoliar", "basto", + "scapuloradial", "suspend", + "stiffish", "Sphenodontidae", + "eternal", "verbid", + "mammonish", "upcushion", + "barkometer", "concretion", + "preagitate", "incomprehensible", + "tristich", "visceral", + "hemimelus", "patroller", + "stentorophonic", "pinulus", + "kerykeion", "brutism", + "monstership", "merciful", + "overinstruct", "defensibly", + "bettermost", "splenauxe", + "Mormyrus", "unreprimanded", + "taver", "ell", + "proacquittal", "infestation", + "overwoven", "Lincolnlike", + "chacona", "Tamil", + "classificational", "lebensraum", + "reeveland", "intuition", + "Whilkut", "focaloid", + "Eleusinian", "micromembrane", + "byroad", "nonrepetition", + "bacterioblast", "brag", + "ribaldrous", "phytoma", + "counteralliance", "pelvimetry", + "pelf", "relaster", + "thermoresistant", "aneurism", + "molossic", "euphonym", + "upswell", "ladhood", + "phallaceous", "inertly", + "gunshop", "stereotypography", + "laryngic", "refasten", + "twinling", "oflete", + "hepatorrhaphy", "electrotechnics", + "cockal", "guitarist", + "topsail", "Cimmerianism", + "larklike", "Llandovery", + "pyrocatechol", "immatchable", + "chooser", "metrocratic", + "craglike", "quadrennial", + "nonpoisonous", "undercolored", + "knob", "ultratense", + "balladmonger", "slait", + "sialadenitis", "bucketer", + "magnificently", "unstipulated", + "unscourged", "unsupercilious", + "packsack", "pansophism", + "soorkee", "percent", + "subirrigate", "champer", + "metapolitics", "spherulitic", + "involatile", "metaphonical", + "stachyuraceous", "speckedness", + "bespin", "proboscidiform", + "gul", "squit", + "yeelaman", "peristeropode", + "opacousness", "shibuichi", + "retinize", "yote", + "misexposition", "devilwise", + "pumpkinification", "vinny", + "bonze", "glossing", + "decardinalize", "transcortical", + "serphoid", "deepmost", + "guanajuatite", "wemless", + "arval", "lammy", + "Effie", "Saponaria", + "tetrahedral", "prolificy", + "excerpt", "dunkadoo", + "Spencerism", "insatiately", + "Gilaki", "oratorship", + "arduousness", "unbashfulness", + "Pithecolobium", "unisexuality", + "veterinarian", "detractive", + "liquidity", "acidophile", + "proauction", "sural", + "totaquina", "Vichyite", + "uninhabitedness", "allegedly", + "Gothish", "manny", + "Inger", "flutist", + "ticktick", "Ludgatian", + "homotransplant", "orthopedical", + "diminutively", "monogoneutic", + "Kenipsim", "sarcologist", + "drome", "stronghearted", + "Fameuse", "Swaziland", + "alen", "chilblain", + "beatable", "agglomeratic", + "constitutor", "tendomucoid", + "porencephalous", "arteriasis", + "boser", "tantivy", + "rede", "lineamental", + "uncontradictableness", "homeotypical", + "masa", "folious", + "dosseret", "neurodegenerative", + "subtransverse", "Chiasmodontidae", + "palaeotheriodont", "unstressedly", + "chalcites", "piquantness", + "lampyrine", "Aplacentalia", + "projecting", "elastivity", + "isopelletierin", "bladderwort", + "strander", "almud", + "iniquitously", "theologal", + "bugre", "chargeably", + "imperceptivity", "meriquinoidal", + "mesophyte", "divinator", + "perfunctory", "counterappellant", + "synovial", "charioteer", + "crystallographical", "comprovincial", + "infrastapedial", "pleasurehood", + "inventurous", "ultrasystematic", + "subangulated", "supraoesophageal", + "Vaishnavism", "transude", + "chrysochrous", "ungrave", + "reconciliable", "uninterpleaded", + "erlking", "wherefrom", + "aprosopia", "antiadiaphorist", + "metoxazine", "incalculable", + "umbellic", "predebit", + "foursquare", "unimmortal", + "nonmanufacture", "slangy", + "predisputant", "familist", + "preaffiliate", "friarhood", + "corelysis", "zoonitic", + "halloo", "paunchy", + "neuromimesis", "aconitine", + "hackneyed", "unfeeble", + "cubby", "autoschediastical", + "naprapath", "lyrebird", + "inexistency", "leucophoenicite", + "ferrogoslarite", "reperuse", + "uncombable", "tambo", + "propodiale", "diplomatize", + "Russifier", "clanned", + "corona", "michigan", + "nonutilitarian", "transcorporeal", + "bought", "Cercosporella", + "stapedius", "glandularly", + "pictorially", "weism", + "disilane", "rainproof", + "Caphtor", "scrubbed", + "oinomancy", "pseudoxanthine", + "nonlustrous", "redesertion", + "Oryzorictinae", "gala", + "Mycogone", "reappreciate", + "cyanoguanidine", "seeingness", + "breadwinner", "noreast", + "furacious", "epauliere", + "omniscribent", "Passiflorales", + "uninductive", "inductivity", + "Orbitolina", "Semecarpus", + "migrainoid", "steprelationship", + "phlogisticate", "mesymnion", + "sloped", "edificator", + "beneficent", "culm", + "paleornithology", "unurban", + "throbless", "amplexifoliate", + "sesquiquintile", "sapience", + "astucious", "dithery", + "boor", "ambitus", + "scotching", "uloid", + "uncompromisingness", "hoove", + "waird", "marshiness", + "Jerusalem", "mericarp", + "unevoked", "benzoperoxide", + "outguess", "pyxie", + "hymnic", "euphemize", + "mendacity", "erythremia", + "rosaniline", "unchatteled", + "lienteria", "Bushongo", + "dialoguer", "unrepealably", + "rivethead", "antideflation", + "vinegarish", "manganosiderite", + "doubtingness", "ovopyriform", + "Cephalodiscus", "Muscicapa", + "Animalivora", "angina", + "planispheric", "ipomoein", + "cuproiodargyrite", "sandbox", + "scrat", "Munnopsidae", + "shola", "pentafid", + "overstudiousness", "times", + "nonprofession", "appetible", + "valvulotomy", "goladar", + "uniarticular", "oxyterpene", + "unlapsing", "omega", + "trophonema", "seminonflammable", + "circumzenithal", "starer", + "depthwise", "liberatress", + "unleavened", "unrevolting", + "groundneedle", "topline", + "wandoo", "umangite", + "ordinant", "unachievable", + "oversand", "snare", + "avengeful", "unexplicit", + "mustafina", "sonable", + "rehabilitative", "eulogization", + "papery", "technopsychology", + "impressor", "cresylite", + "entame", "transudatory", + "scotale", "pachydermatoid", + "imaginary", "yeat", + "slipped", "stewardship", + "adatom", "cockstone", + "skyshine", "heavenful", + "comparability", "exprobratory", + "dermorhynchous", "parquet", + "cretaceous", "vesperal", + "raphis", "undangered", + "Glecoma", "engrain", + "counteractively", "Zuludom", + "orchiocatabasis", "Auriculariales", + "warriorwise", "extraorganismal", + "overbuilt", "alveolite", + "tetchy", "terrificness", + "widdle", "unpremonished", + "rebilling", "sequestrum", + "equiconvex", "heliocentricism", + "catabaptist", "okonite", + "propheticism", "helminthagogic", + "calycular", "giantly", + "wingable", "golem", + "unprovided", "commandingness", + "greave", "haply", + "doina", "depressingly", + "subdentate", "impairment", + "decidable", "neurotrophic", + "unpredict", "bicorporeal", + "pendulant", "flatman", + "intrabred", "toplike", + "Prosobranchiata", "farrantly", + "toxoplasmosis", "gorilloid", + "dipsomaniacal", "aquiline", + "atlantite", "ascitic", + "perculsive", "prospectiveness", + "saponaceous", "centrifugalization", + "dinical", "infravaginal", + "beadroll", "affaite", + "Helvidian", "tickleproof", + "abstractionism", "enhedge", + "outwealth", "overcontribute", + "coldfinch", "gymnastic", + "Pincian", "Munychian", + "codisjunct", "quad", + "coracomandibular", "phoenicochroite", + "amender", "selectivity", + "putative", "semantician", + "lophotrichic", "Spatangoidea", + "saccharogenic", "inferent", + "Triconodonta", "arrendation", + "sheepskin", "taurocolla", + "bunghole", "Machiavel", + "triakistetrahedral", "dehairer", + "prezygapophysial", "cylindric", + "pneumonalgia", "sleigher", + "emir", "Socraticism", + "licitness", "massedly", + "instructiveness", "sturdied", + "redecrease", "starosta", + "evictor", "orgiastic", + "squdge", "meloplasty", + "Tsonecan", "repealableness", + "swoony", "myesthesia", + "molecule", "autobiographist", + "reciprocation", "refective", + "unobservantness", "tricae", + "ungouged", "floatability", + "Mesua", "fetlocked", + "chordacentrum", "sedentariness", + "various", "laubanite", + "nectopod", "zenick", + "sequentially", "analgic", + "biodynamics", "posttraumatic", + "nummi", "pyroacetic", + "bot", "redescend", + "dispermy", "undiffusive", + "circular", "trillion", + "Uraniidae", "ploration", + "discipular", "potentness", + "sud", "Hu", + "Eryon", "plugger", + "subdrainage", "jharal", + "abscission", "supermarket", + "countergabion", "glacierist", + "lithotresis", "minniebush", + "zanyism", "eucalypteol", + "sterilely", "unrealize", + "unpatched", "hypochondriacism", + "critically", "cheesecutter", + }; } diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomWriter.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomWriter.java index 1627d688ab..506dd79865 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomWriter.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/RandomWriter.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -74,44 +74,44 @@ * 1099511627776 * * - * + * * Equivalently, {@link RandomWriter} also supports all the above options * and ones supported by {@link GenericOptionsParser} via the command-line. */ public class RandomWriter extends Configured implements Tool { public static final String TOTAL_BYTES = "mapreduce.randomwriter.totalbytes"; - public static final String BYTES_PER_MAP = - "mapreduce.randomwriter.bytespermap"; - public static final String MAPS_PER_HOST = - "mapreduce.randomwriter.mapsperhost"; + public static final String BYTES_PER_MAP = + "mapreduce.randomwriter.bytespermap"; + public static final String MAPS_PER_HOST = + "mapreduce.randomwriter.mapsperhost"; public static final String MAX_VALUE = "mapreduce.randomwriter.maxvalue"; public static final String MIN_VALUE = "mapreduce.randomwriter.minvalue"; public static final String MIN_KEY = "mapreduce.randomwriter.minkey"; public static final String MAX_KEY = "mapreduce.randomwriter.maxkey"; - + /** * User counters */ - static enum Counters { RECORDS_WRITTEN, BYTES_WRITTEN } - + static enum Counters {RECORDS_WRITTEN, BYTES_WRITTEN} + /** * A custom input format that creates virtual inputs of a single string * for each map. */ static class RandomInputFormat extends InputFormat { - /** + /** * Generate the requested number of file splits, with the filename * set to the filename of the output file. */ public List getSplits(JobContext job) throws IOException { List result = new ArrayList(); Path outDir = FileOutputFormat.getOutputPath(job); - int numSplits = - job.getConfiguration().getInt(MRJobConfig.NUM_MAPS, 1); - for(int i=0; i < numSplits; ++i) { - result.add(new FileSplit(new Path(outDir, "dummy-split-" + i), 0, 1, - (String[])null)); + int numSplits = + job.getConfiguration().getInt(MRJobConfig.NUM_MAPS, 1); + for (int i = 0; i < numSplits; ++i) { + result.add(new FileSplit(new Path(outDir, "dummy-split-" + i), 0, 1, + (String[]) null)); } return result; } @@ -124,16 +124,17 @@ static class RandomRecordReader extends RecordReader { Path name; Text key = null; Text value = new Text(); + public RandomRecordReader(Path p) { name = p; } - + public void initialize(InputSplit split, TaskAttemptContext context) - throws IOException, InterruptedException { - + throws IOException, InterruptedException { + } - + public boolean nextKeyValue() { if (name != null) { key = new Text(); @@ -143,15 +144,15 @@ public boolean nextKeyValue() { } return false; } - + public Text getCurrentKey() { return key; } - + public Text getCurrentValue() { return value; } - + public void close() {} public float getProgress() { @@ -160,14 +161,14 @@ public float getProgress() { } public RecordReader createRecordReader(InputSplit split, - TaskAttemptContext context) throws IOException, InterruptedException { + TaskAttemptContext context) throws IOException, InterruptedException { return new RandomRecordReader(((FileSplit) split).getPath()); } } static class RandomMapper extends Mapper { - + BytesWritable, BytesWritable> { + private long numBytesToWrite; private int minKeySize; private int keySizeRange; @@ -176,27 +177,27 @@ static class RandomMapper extends Mapper= offset; --i) { + for (int i = offset + length - 1; i >= offset; --i) { data[i] = (byte) random.nextInt(256); } } - + /** * Given an output filename, write a bunch of random records to it. */ - public void map(WritableComparable key, + public void map(WritableComparable key, Writable value, - Context context) throws IOException,InterruptedException { + Context context) throws IOException, InterruptedException { int itemCount = 0; while (numBytesToWrite > 0) { - int keyLength = minKeySize + - (keySizeRange != 0 ? random.nextInt(keySizeRange) : 0); + int keyLength = minKeySize + + (keySizeRange != 0 ? random.nextInt(keySizeRange) : 0); randomKey.setSize(keyLength); randomizeBytes(randomKey.getBytes(), 0, randomKey.getLength()); int valueLength = minValueSize + - (valueSizeRange != 0 ? random.nextInt(valueSizeRange) : 0); + (valueSizeRange != 0 ? random.nextInt(valueSizeRange) : 0); randomValue.setSize(valueLength); randomizeBytes(randomValue.getBytes(), 0, randomValue.getLength()); context.write(randomKey, randomValue); @@ -204,13 +205,13 @@ public void map(WritableComparable key, context.getCounter(Counters.BYTES_WRITTEN).increment(keyLength + valueLength); context.getCounter(Counters.RECORDS_WRITTEN).increment(1); if (++itemCount % 200 == 0) { - context.setStatus("wrote record " + itemCount + ". " + - numBytesToWrite + " bytes left."); + context.setStatus("wrote record " + itemCount + ". " + + numBytesToWrite + " bytes left."); } } context.setStatus("done with " + itemCount + " records."); } - + /** * Save the values out of the configuaration that we need to write * the data. @@ -219,44 +220,44 @@ public void map(WritableComparable key, public void setup(Context context) { Configuration conf = context.getConfiguration(); numBytesToWrite = conf.getLong(BYTES_PER_MAP, - 1*1024*1024*1024); + 1 * 1024 * 1024 * 1024); minKeySize = conf.getInt(MIN_KEY, 10); - keySizeRange = - conf.getInt(MAX_KEY, 1000) - minKeySize; + keySizeRange = + conf.getInt(MAX_KEY, 1000) - minKeySize; minValueSize = conf.getInt(MIN_VALUE, 0); - valueSizeRange = - conf.getInt(MAX_VALUE, 20000) - minValueSize; + valueSizeRange = + conf.getInt(MAX_VALUE, 20000) - minValueSize; } } - + /** * This is the main routine for launching a distributed random write job. * It runs 10 maps/node and each node writes 1 gig of data to a DFS file. * The reduce doesn't do anything. - * - * @throws IOException + * + * @throws IOException */ @SuppressWarnings("deprecation") - public int run(String[] args) throws Exception { + public int run(String[] args) throws Exception { if (args.length == 0) { System.out.println("Usage: writer "); ToolRunner.printGenericCommandUsage(System.out); return 2; } - + Path outDir = new Path(args[0]); Configuration conf = getConf(); JobClient client = new JobClient(conf); ClusterStatus cluster = client.getClusterStatus(); int numMapsPerHost = conf.getInt(MAPS_PER_HOST, 10); long numBytesToWritePerMap = conf.getLong(BYTES_PER_MAP, - 1*1024*1024*1024); + 1 * 1024 * 1024 * 1024); if (numBytesToWritePerMap == 0) { System.err.println("Cannot have" + BYTES_PER_MAP + " set to 0"); return -2; } - long totalBytesToWrite = conf.getLong(TOTAL_BYTES, - numMapsPerHost*numBytesToWritePerMap*cluster.getTaskTrackers()); + long totalBytesToWrite = conf.getLong(TOTAL_BYTES, + numMapsPerHost * numBytesToWritePerMap * cluster.getTaskTrackers()); int numMaps = (int) (totalBytesToWrite / numBytesToWritePerMap); if (numMaps == 0 && totalBytesToWrite > 0) { numMaps = 1; @@ -265,37 +266,36 @@ public int run(String[] args) throws Exception { conf.setInt(MRJobConfig.NUM_MAPS, numMaps); Job job = new Job(conf); - + job.setJarByClass(RandomWriter.class); job.setJobName("random-writer"); FileOutputFormat.setOutputPath(job, outDir); job.setOutputKeyClass(BytesWritable.class); job.setOutputValueClass(BytesWritable.class); job.setInputFormatClass(RandomInputFormat.class); - job.setMapperClass(RandomMapper.class); + job.setMapperClass(RandomMapper.class); job.setReducerClass(Reducer.class); job.setOutputFormatClass(SequenceFileOutputFormat.class); - + System.out.println("Running " + numMaps + " maps."); - + // reducer NONE job.setNumReduceTasks(0); - + Date startTime = new Date(); System.out.println("Job started: " + startTime); int ret = job.waitForCompletion(true) ? 0 : 1; Date endTime = new Date(); System.out.println("Job ended: " + endTime); - System.out.println("The job took " + - (endTime.getTime() - startTime.getTime()) /1000 + - " seconds."); - + System.out.println("The job took " + + (endTime.getTime() - startTime.getTime()) / 1000 + + " seconds."); + return ret; } - + public static void main(String[] args) throws Exception { int res = ToolRunner.run(getTezDecoratedConfiguration(), new RandomWriter(), args); System.exit(res); } - } diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/SecondarySort.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/SecondarySort.java index f42d08a505..6fc2b3b75e 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/SecondarySort.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/SecondarySort.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -52,16 +52,16 @@ * in-dir out-dir */ public class SecondarySort extends Configured implements Tool { - + /** * Define a pair of integers that are writable. * They are serialized in a byte comparable format. */ - public static class IntPair - implements WritableComparable { + public static class IntPair + implements WritableComparable { private int first = 0; private int second = 0; - + /** * Set the left and right values. */ @@ -69,12 +69,15 @@ public void set(int left, int right) { first = left; second = right; } + public int getFirst() { return first; } + public int getSecond() { return second; } + /** * Read the two integers. * Encoded as: MIN_VALUE -> 0, 0 -> -MIN_VALUE, MAX_VALUE-> -1 @@ -84,15 +87,18 @@ public void readFields(DataInput in) throws IOException { first = in.readInt() + Integer.MIN_VALUE; second = in.readInt() + Integer.MIN_VALUE; } + @Override public void write(DataOutput out) throws IOException { out.writeInt(first - Integer.MIN_VALUE); out.writeInt(second - Integer.MIN_VALUE); } + @Override public int hashCode() { return first * 157 + second; } + @Override public boolean equals(Object right) { if (right instanceof IntPair) { @@ -102,7 +108,8 @@ public boolean equals(Object right) { return false; } } - /** A Comparator that compares serialized IntPair. */ + + /** A Comparator that compares serialized IntPair. */ public static class Comparator extends WritableComparator { public Comparator() { super(IntPair.class); @@ -129,13 +136,13 @@ public int compareTo(IntPair o) { } } } - + /** * Partition based on the first part of the pair. */ - public static class FirstPartitioner extends Partitioner{ + public static class FirstPartitioner extends Partitioner { @Override - public int getPartition(IntPair key, IntWritable value, + public int getPartition(IntPair key, IntWritable value, int numPartitions) { return Math.abs(key.getFirst() * 127) % numPartitions; } @@ -145,12 +152,12 @@ public int getPartition(IntPair key, IntWritable value, * Compare only the first part of the pair, so that reduce is called once * for each value of the first part. */ - public static class FirstGroupingComparator - implements RawComparator { + public static class FirstGroupingComparator + implements RawComparator { @Override public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { - return WritableComparator.compareBytes(b1, s1, Integer.SIZE/8, - b2, s2, Integer.SIZE/8); + return WritableComparator.compareBytes(b1, s1, Integer.SIZE / 8, + b2, s2, Integer.SIZE / 8); } @Override @@ -165,14 +172,14 @@ public int compare(IntPair o1, IntPair o2) { * Read two integers from each line and generate a key, value pair * as ((left, right), right). */ - public static class MapClass - extends Mapper { - + public static class MapClass + extends Mapper { + private final IntPair key = new IntPair(); private final IntWritable value = new IntWritable(); - + @Override - public void map(LongWritable inKey, Text inValue, + public void map(LongWritable inKey, Text inValue, Context context) throws IOException, InterruptedException { StringTokenizer itr = new StringTokenizer(inValue.toString()); int left = 0; @@ -188,23 +195,23 @@ public void map(LongWritable inKey, Text inValue, } } } - + /** * A reducer class that just emits the sum of the input values. */ - public static class Reduce - extends Reducer { - private static final Text SEPARATOR = - new Text("------------------------------------------------"); + public static class Reduce + extends Reducer { + private static final Text SEPARATOR = + new Text("------------------------------------------------"); private final Text first = new Text(); - + @Override public void reduce(IntPair key, Iterable values, Context context - ) throws IOException, InterruptedException { + ) throws IOException, InterruptedException { context.write(SEPARATOR, null); first.set(Integer.toString(key.getFirst())); - for(IntWritable value: values) { + for (IntWritable value : values) { context.write(first, value); } } @@ -236,7 +243,7 @@ public int run(String[] args) throws Exception { // the reduce output is Text, IntWritable job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); - + FileInputFormat.addInputPath(job, new Path(otherArgs[0])); FileOutputFormat.setOutputPath(job, new Path(otherArgs[1])); return job.waitForCompletion(true) ? 0 : 1; diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Sort.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Sort.java index 9a08b74a2d..36b2f477da 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Sort.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/Sort.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -57,19 +57,19 @@ * [-totalOrder pcnt num samples max splits] * in-dir out-dir */ -public class Sort extends Configured implements Tool { - public static final String REDUCES_PER_HOST = - "mapreduce.sort.reducesperhost"; +public class Sort extends Configured implements Tool { + public static final String REDUCES_PER_HOST = + "mapreduce.sort.reducesperhost"; private Job job = null; static int printUsage() { System.out.println("sort [-r ] " + - "[-inFormat ] " + - "[-outFormat ] " + - "[-outKey ] " + - "[-outValue ] " + - "[-totalOrder ] " + - " "); + "[-inFormat ] " + + "[-outFormat ] " + + "[-outKey ] " + + "[-outValue ] " + + "[-totalOrder ] " + + " "); ToolRunner.printGenericCommandUsage(System.out); return 2; } @@ -89,40 +89,40 @@ public int run(String[] args) throws Exception { int num_reduces = (int) (cluster.getMaxReduceTasks() * 0.9); String sort_reduces = conf.get(REDUCES_PER_HOST); if (sort_reduces != null) { - num_reduces = cluster.getTaskTrackers() * - Integer.parseInt(sort_reduces); + num_reduces = cluster.getTaskTrackers() * + Integer.parseInt(sort_reduces); } - Class inputFormatClass = - SequenceFileInputFormat.class; - Class outputFormatClass = - SequenceFileOutputFormat.class; + Class inputFormatClass = + SequenceFileInputFormat.class; + Class outputFormatClass = + SequenceFileOutputFormat.class; Class outputKeyClass = BytesWritable.class; Class outputValueClass = BytesWritable.class; List otherArgs = new ArrayList(); - InputSampler.Sampler sampler = null; - for(int i=0; i < args.length; ++i) { + InputSampler.Sampler sampler = null; + for (int i = 0; i < args.length; ++i) { try { if ("-r".equals(args[i])) { num_reduces = Integer.parseInt(args[++i]); } else if ("-inFormat".equals(args[i])) { - inputFormatClass = - Class.forName(args[++i]).asSubclass(InputFormat.class); + inputFormatClass = + Class.forName(args[++i]).asSubclass(InputFormat.class); } else if ("-outFormat".equals(args[i])) { - outputFormatClass = - Class.forName(args[++i]).asSubclass(OutputFormat.class); + outputFormatClass = + Class.forName(args[++i]).asSubclass(OutputFormat.class); } else if ("-outKey".equals(args[i])) { - outputKeyClass = - Class.forName(args[++i]).asSubclass(WritableComparable.class); + outputKeyClass = + Class.forName(args[++i]).asSubclass(WritableComparable.class); } else if ("-outValue".equals(args[i])) { - outputValueClass = - Class.forName(args[++i]).asSubclass(Writable.class); + outputValueClass = + Class.forName(args[++i]).asSubclass(Writable.class); } else if ("-totalOrder".equals(args[i])) { double pcnt = Double.parseDouble(args[++i]); int numSamples = Integer.parseInt(args[++i]); int maxSplits = Integer.parseInt(args[++i]); if (0 >= maxSplits) maxSplits = Integer.MAX_VALUE; sampler = - new InputSampler.RandomSampler(pcnt, numSamples, maxSplits); + new InputSampler.RandomSampler(pcnt, numSamples, maxSplits); } else { otherArgs.add(args[i]); } @@ -131,7 +131,7 @@ public int run(String[] args) throws Exception { return printUsage(); } catch (ArrayIndexOutOfBoundsException except) { System.out.println("ERROR: Required parameter missing from " + - args[i-1]); + args[i - 1]); return printUsage(); // exits } } @@ -140,7 +140,7 @@ public int run(String[] args) throws Exception { job.setJobName("sorter"); job.setJarByClass(Sort.class); - job.setMapperClass(Mapper.class); + job.setMapperClass(Mapper.class); job.setReducerClass(Reducer.class); job.setNumReduceTasks(num_reduces); @@ -159,7 +159,7 @@ public int run(String[] args) throws Exception { } FileInputFormat.setInputPaths(job, otherArgs.get(0)); FileOutputFormat.setOutputPath(job, new Path(otherArgs.get(1))); - + if (sampler != null) { System.out.println("Sampling input to effect total-order sort..."); job.setPartitionerClass(TotalOrderPartitioner.class); @@ -167,15 +167,15 @@ public int run(String[] args) throws Exception { inputDir = inputDir.makeQualified(inputDir.getFileSystem(conf)); Path partitionFile = new Path(inputDir, "_sortPartitioning"); TotalOrderPartitioner.setPartitionFile(conf, partitionFile); - InputSampler.writePartitionFile(job, sampler); + InputSampler.writePartitionFile(job, sampler); URI partitionUri = new URI(partitionFile.toString() + - "#" + "_sortPartitioning"); + "#" + "_sortPartitioning"); DistributedCache.addCacheFile(partitionUri, conf); } System.out.println("Running on " + cluster.getTaskTrackers() + - " nodes to sort from " + + " nodes to sort from " + FileInputFormat.getInputPaths(job)[0] + " into " + FileOutputFormat.getOutputPath(job) + " with " + num_reduces + " reduces."); @@ -184,13 +184,11 @@ public int run(String[] args) throws Exception { int ret = job.waitForCompletion(true) ? 0 : 1; Date end_time = new Date(); System.out.println("Job ended: " + end_time); - System.out.println("The job took " + - (end_time.getTime() - startTime.getTime()) /1000 + " seconds."); + System.out.println("The job took " + + (end_time.getTime() - startTime.getTime()) / 1000 + " seconds."); return ret; } - - public static void main(String[] args) throws Exception { int res = ToolRunner.run(getTezDecoratedConfiguration(), new Sort(), args); System.exit(res); diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java index 1b87e11713..3faa761952 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -117,7 +117,7 @@ public class TestOrderedWordCount extends Configured implements Tool { private static final int NO_OF_VERTICES = 3; public static class TokenizerMapper - extends Mapper{ + extends Mapper { private final static IntWritable one = new IntWritable(1); private Text word = new Text(); @@ -130,12 +130,12 @@ public void setup(Context context) throws IOException, InterruptedException { " Total Dag Payload sent through IPC : " + (conf.getInt(MAX_IPC_DATA_LENGTH, -1) + conf.getInt(EXCEED_IPC_DATA_LIMIT, 3)) + " MB," + " Each Vertex Processor payload : " + - ((conf.getInt(MAX_IPC_DATA_LENGTH, -1) + conf.getInt(EXCEED_IPC_DATA_LIMIT, 3))/NO_OF_VERTICES)+" MB"); + ((conf.getInt(MAX_IPC_DATA_LENGTH, -1) + conf.getInt(EXCEED_IPC_DATA_LIMIT, 3)) / NO_OF_VERTICES) + " MB"); } } public void map(Object key, Text value, Context context - ) throws IOException, InterruptedException { + ) throws IOException, InterruptedException { StringTokenizer itr = new StringTokenizer(value.toString()); while (itr.hasMoreTokens()) { word.set(itr.nextToken()); @@ -145,12 +145,12 @@ public void map(Object key, Text value, Context context } public static class IntSumReducer - extends Reducer { + extends Reducer { private IntWritable result = new IntWritable(); public void reduce(Text key, Iterable values, Context context - ) throws IOException, InterruptedException { + ) throws IOException, InterruptedException { int sum = 0; for (IntWritable val : values) { sum += val.get(); @@ -169,8 +169,8 @@ public static class MyOrderByNoOpReducer extends Reducer { public void reduce(IntWritable key, Iterable values, - Context context - ) throws IOException, InterruptedException { + Context context + ) throws IOException, InterruptedException { for (Text word : values) { context.write(word, key); } @@ -181,13 +181,13 @@ public void reduce(IntWritable key, Iterable values, @VisibleForTesting public DAG createDAG(FileSystem fs, Configuration conf, - Map commonLocalResources, Path stagingDir, - int dagIndex, String inputPath, String outputPath, - boolean generateSplitsInClient, - boolean useMRSettings, - int intermediateNumReduceTasks, - int maxDataLengthThroughIPC, - int exceedDataLimit) throws Exception { + Map commonLocalResources, Path stagingDir, + int dagIndex, String inputPath, String outputPath, + boolean generateSplitsInClient, + boolean useMRSettings, + int intermediateNumReduceTasks, + int maxDataLengthThroughIPC, + int exceedDataLimit) throws Exception { Configuration mapStageConf = new JobConf(conf); mapStageConf.set(MRJobConfig.MAP_CLASS_ATTR, @@ -268,7 +268,7 @@ public DAG createDAG(FileSystem fs, Configuration conf, String iReduceStageHistoryText = TezUtils.convertToHistoryText("Intermediate Summation Vertex", iReduceStageConf); ProcessorDescriptor iReduceProcessorDescriptor = ProcessorDescriptor.create( - ReduceProcessor.class.getName()) + ReduceProcessor.class.getName()) .setUserPayload(TezUtils.createUserPayloadFromConf(copyiReduceStageConf)) .setHistoryText(iReduceStageHistoryText); @@ -295,7 +295,7 @@ public DAG createDAG(FileSystem fs, Configuration conf, ProcessorDescriptor finalReduceProcessorDescriptor = ProcessorDescriptor.create( - ReduceProcessor.class.getName()) + ReduceProcessor.class.getName()) .setUserPayload(finalReducePayload) .setHistoryText(finalReduceStageHistoryText); if (!useMRSettings) { @@ -322,7 +322,7 @@ public DAG createDAG(FileSystem fs, Configuration conf, OrderedPartitionedKVEdgeConfig edgeConf1 = OrderedPartitionedKVEdgeConfig .newBuilder(Text.class.getName(), IntWritable.class.getName(), HashPartitioner.class.getName()).setFromConfiguration(iReduceStageConf) - .configureInput().useLegacyInput().done().build(); + .configureInput().useLegacyInput().done().build(); dag.addEdge( Edge.create(dag.getVertex("initialmap"), dag.getVertex("intermediate_reducer"), edgeConf1.createDefaultEdgeProperty())); @@ -330,7 +330,7 @@ public DAG createDAG(FileSystem fs, Configuration conf, OrderedPartitionedKVEdgeConfig edgeConf2 = OrderedPartitionedKVEdgeConfig .newBuilder(IntWritable.class.getName(), Text.class.getName(), HashPartitioner.class.getName()).setFromConfiguration(finalReduceConf) - .configureInput().useLegacyInput().done().build(); + .configureInput().useLegacyInput().done().build(); dag.addEdge( Edge.create(dag.getVertex("intermediate_reducer"), dag.getVertex("finalreduce"), edgeConf2.createDefaultEdgeProperty())); @@ -339,6 +339,7 @@ public DAG createDAG(FileSystem fs, Configuration conf, return dag; } + private void setMaxDataLengthConf(Configuration config, int maxDataLengthThroughIPC, int exceedDataLimit) { /** * if -Dtez.testorderedwordcount.ipc.maximum.data.length is set by user, @@ -366,9 +367,8 @@ private void updateDAGACls(Configuration conf, DAG dag, int dagIndex) { || conf.get(DAG_MODIFY_ACLS + suffix) != null) { accessControls = new DAGAccessControls( conf.get(DAG_VIEW_ACLS + suffix), conf.get(DAG_MODIFY_ACLS + suffix)); - } else if (conf.get(DAG_VIEW_ACLS) != null - || conf.get(DAG_MODIFY_ACLS) != null) { + || conf.get(DAG_MODIFY_ACLS) != null) { accessControls = new DAGAccessControls( conf.get(DAG_VIEW_ACLS), conf.get(DAG_MODIFY_ACLS)); } @@ -378,7 +378,6 @@ private void updateDAGACls(Configuration conf, DAG dag, int dagIndex) { } } - private static void printUsage() { String options = " [-generateSplitsInClient true/]"; System.err.println("Usage: testorderedwordcount " + options); @@ -387,7 +386,6 @@ private static void printUsage() { ToolRunner.printGenericCommandUsage(System.err); } - @Override public int run(String[] args) throws Exception { Configuration conf = getConf(); @@ -419,7 +417,7 @@ public int run(String[] args) throws Exception { conf.setInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, maxDataLengthThroughIPC * 1024 * 1024); } - if (((otherArgs.length%2) != 0) + if (((otherArgs.length % 2) != 0) || (!useTezSession && otherArgs.length != 2)) { printUsage(); return 2; @@ -429,28 +427,28 @@ public int run(String[] args) throws Exception { List outputPaths = new ArrayList(); TezConfiguration tezConf = new TezConfiguration(conf); - for (int i = 0; i < otherArgs.length; i+=2) { + for (int i = 0; i < otherArgs.length; i += 2) { FileSystem inputPathFs = new Path(otherArgs[i]).getFileSystem(tezConf); inputPaths.add(inputPathFs.makeQualified(new Path(otherArgs[i])).toString()); - FileSystem outputPathFs = new Path(otherArgs[i+1]).getFileSystem(tezConf); - outputPaths.add(outputPathFs.makeQualified(new Path(otherArgs[i+1])).toString()); + FileSystem outputPathFs = new Path(otherArgs[i + 1]).getFileSystem(tezConf); + outputPaths.add(outputPathFs.makeQualified(new Path(otherArgs[i + 1])).toString()); } UserGroupInformation.setConfiguration(conf); HadoopShim hadoopShim = new HadoopShimsLoader(tezConf).getHadoopShim(); TestOrderedWordCount instance = new TestOrderedWordCount(); - String stagingDirStr = conf.get(TezConfiguration.TEZ_AM_STAGING_DIR, - TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT) + Path.SEPARATOR + - Long.toString(System.currentTimeMillis()); + String stagingDirStr = conf.get(TezConfiguration.TEZ_AM_STAGING_DIR, + TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT) + Path.SEPARATOR + + Long.toString(System.currentTimeMillis()); Path stagingDir = new Path(stagingDirStr); FileSystem pathFs = stagingDir.getFileSystem(tezConf); pathFs.mkdirs(new Path(stagingDirStr)); tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirStr); stagingDir = pathFs.makeQualified(new Path(stagingDirStr)); - - TokenCache.obtainTokensForNamenodes(instance.credentials, new Path[] {stagingDir}, conf); + + TokenCache.obtainTokensForNamenodes(instance.credentials, new Path[]{stagingDir}, conf); TezClientUtils.ensureStagingDirExists(tezConf, stagingDir); // No need to add jar containing this class as assumed to be part of @@ -458,7 +456,7 @@ public int run(String[] args) throws Exception { // TEZ-674 Obtain tokens based on the Input / Output paths. For now assuming staging dir // is the same filesystem as the one used for Input/Output. - + if (useTezSession) { LOG.info("Creating Tez Session"); tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true); @@ -474,8 +472,8 @@ public int run(String[] args) throws Exception { DAGStatus dagStatus = null; DAGClient dagClient = null; - String[] vNames = { "initialmap", "intermediate_reducer", - "finalreduce" }; + String[] vNames = {"initialmap", "intermediate_reducer", + "finalreduce"}; Set statusGetOpts = EnumSet.of(StatusGetOpts.GET_COUNTERS); try { @@ -484,7 +482,7 @@ public int run(String[] args) throws Exception { && interJobSleepTimeout > 0) { try { LOG.info("Sleeping between jobs, sleepInterval=" - + (interJobSleepTimeout/1000)); + + (interJobSleepTimeout / 1000)); Thread.sleep(interJobSleepTimeout); } catch (InterruptedException e) { LOG.info("Main thread interrupted. Breaking out of job loop"); @@ -492,8 +490,8 @@ public int run(String[] args) throws Exception { } } - String inputPath = inputPaths.get(dagIndex-1); - String outputPath = outputPaths.get(dagIndex-1); + String inputPath = inputPaths.get(dagIndex - 1); + String outputPath = outputPaths.get(dagIndex - 1); Path outputPathAsPath = new Path(outputPath); FileSystem fs = outputPathAsPath.getFileSystem(conf); @@ -508,16 +506,16 @@ public int run(String[] args) throws Exception { + ", outputPath=" + outputPath); Map localResources = - new TreeMap(); - + new TreeMap(); + DAG dag = instance.createDAG(fs, tezConf, localResources, stagingDir, dagIndex, inputPath, outputPath, generateSplitsInClient, useMRSettings, intermediateNumReduceTasks, - maxDataLengthThroughIPC,exceedDataLimit); + maxDataLengthThroughIPC, exceedDataLimit); String callerType = "TestOrderedWordCount"; String callerId = tezSession.getAppMasterApplicationId() == null ? - ( "UnknownApp_" + System.currentTimeMillis() + dagIndex ) : - ( tezSession.getAppMasterApplicationId().toString() + "_" + dagIndex); + ("UnknownApp_" + System.currentTimeMillis() + dagIndex) : + (tezSession.getAppMasterApplicationId().toString() + "_" + dagIndex); dag.setCallerContext(CallerContext.create("Tez", callerId, callerType, "TestOrderedWordCount Job")); @@ -537,7 +535,7 @@ public int run(String[] args) throws Exception { preWarmVertex.addTaskLocalFiles(dag.getVertex("initialmap").getTaskLocalFiles()); preWarmVertex.setTaskEnvironment(dag.getVertex("initialmap").getTaskEnvironment()); preWarmVertex.setTaskLaunchCmdOpts(dag.getVertex("initialmap").getTaskLaunchCmdOpts()); - + tezSession.preWarm(preWarmVertex); } @@ -568,7 +566,6 @@ public int run(String[] args) throws Exception { } } - while (dagStatus.getState() != DAGStatus.State.SUCCEEDED && dagStatus.getState() != DAGStatus.State.FAILED && dagStatus.getState() != DAGStatus.State.KILLED && @@ -594,7 +591,7 @@ public int run(String[] args) throws Exception { + "FinalState=" + dagStatus.getState()); if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) { LOG.info("DAG " + dagIndex + " diagnostics: " - + dagStatus.getDiagnostics()); + + dagStatus.getDiagnostics()); } } } catch (Exception e) { @@ -616,7 +613,7 @@ public int run(String[] args) throws Exception { } private static void waitForTezSessionReady(TezClient tezSession) - throws IOException, TezException, InterruptedException { + throws IOException, TezException, InterruptedException { tezSession.waitTillReady(); } diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java index 767691d34b..9f33772358 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -88,7 +88,7 @@ public void run() throws Exception { Preconditions.checkArgument(getOutputs().containsKey("checker")); MRInput input = (MRInput) getInputs().values().iterator().next(); KeyValueReader kvReader = input.getReader(); - Output output = getOutputs().get("checker"); + Output output = getOutputs().get("checker"); KeyValueWriter kvWriter = (KeyValueWriter) output.getWriter(); MROutput parts = null; KeyValueWriter partsWriter = null; @@ -107,7 +107,6 @@ public void run() throws Exception { } } } - } public static class UnionProcessor extends SimpleMRProcessor { @@ -160,14 +159,13 @@ public void run() throws Exception { } kvWriter.write("Union", new IntWritable(unionKv.size())); } - } private DAG createDAG(FileSystem fs, TezConfiguration tezConf, - Map localResources, Path stagingDir, - String inputPath, String outputPath) throws IOException { + Map localResources, Path stagingDir, + String inputPath, String outputPath) throws IOException { DAG dag = DAG.create("UnionExample"); - + int numMaps = -1; Configuration inputConf = new Configuration(tezConf); inputConf.setBoolean("mapred.mapper.new-api", false); @@ -194,14 +192,13 @@ private DAG createDAG(FileSystem fs, TezConfiguration tezConf, outputConf.set(FileOutputFormat.OUTDIR, outputPath); DataSinkDescriptor od = MROutput.createConfigBuilder(outputConf, null).build(); checkerVertex.addDataSink("union", od); - Configuration allPartsConf = new Configuration(tezConf); DataSinkDescriptor od2 = MROutput.createConfigBuilder(allPartsConf, TextOutputFormat.class, outputPath + "-all-parts").build(); checkerVertex.addDataSink("all-parts", od2); - Configuration partsConf = new Configuration(tezConf); + Configuration partsConf = new Configuration(tezConf); DataSinkDescriptor od1 = MROutput.createConfigBuilder(partsConf, TextOutputFormat.class, outputPath + "-parts").build(); VertexGroup unionVertex = dag.createVertexGroup("union", mapVertex1, mapVertex2); @@ -221,7 +218,7 @@ private DAG createDAG(FileSystem fs, TezConfiguration tezConf, GroupInputEdge.create(unionVertex, checkerVertex, edgeConf.createDefaultEdgeProperty(), InputDescriptor.create( ConcatenatedMergedKeyValuesInput.class.getName()))); - return dag; + return dag; } private static void printUsage() { @@ -243,49 +240,48 @@ public boolean run(String inputPath, String outputPath, Configuration conf) thro // staging dir FileSystem fs = FileSystem.get(tezConf); String stagingDirStr = Path.SEPARATOR + "user" + Path.SEPARATOR - + user + Path.SEPARATOR+ ".staging" + Path.SEPARATOR - + Path.SEPARATOR + Long.toString(System.currentTimeMillis()); + + user + Path.SEPARATOR + ".staging" + Path.SEPARATOR + + Path.SEPARATOR + Long.toString(System.currentTimeMillis()); Path stagingDir = new Path(stagingDirStr); tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirStr); stagingDir = fs.makeQualified(stagingDir); - // No need to add jar containing this class as assumed to be part of // the tez jars. // TEZ-674 Obtain tokens based on the Input / Output paths. For now assuming staging dir // is the same filesystem as the one used for Input/Output. - + TezClient tezSession = TezClient.create("UnionExampleSession", tezConf); tezSession.start(); DAGClient dagClient = null; try { - Path outputPathAsPath = new Path(outputPath); + Path outputPathAsPath = new Path(outputPath); FileSystem outputFs = outputPathAsPath.getFileSystem(tezConf); outputPathAsPath = outputFs.makeQualified(outputPathAsPath); - if (outputFs.exists(outputPathAsPath)) { - throw new FileAlreadyExistsException("Output directory " - + outputPath + " already exists"); - } - - Map localResources = + if (outputFs.exists(outputPathAsPath)) { + throw new FileAlreadyExistsException("Output directory " + + outputPath + " already exists"); + } + + Map localResources = new TreeMap(); - - DAG dag = createDAG(fs, tezConf, localResources, - stagingDir, inputPath, outputPath); - - tezSession.waitTillReady(); - dagClient = tezSession.submitDAG(dag); - - // monitoring - DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(EnumSet.of(StatusGetOpts.GET_COUNTERS)); - if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) { - System.out.println("DAG diagnostics: " + dagStatus.getDiagnostics()); - return false; - } - return true; + + DAG dag = createDAG(fs, tezConf, localResources, + stagingDir, inputPath, outputPath); + + tezSession.waitTillReady(); + dagClient = tezSession.submitDAG(dag); + + // monitoring + DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(EnumSet.of(StatusGetOpts.GET_COUNTERS)); + if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) { + System.out.println("DAG diagnostics: " + dagStatus.getDiagnostics()); + return false; + } + return true; } finally { fs.delete(stagingDir, true); tezSession.stop(); diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/helpers/SplitsInClientOptionParser.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/helpers/SplitsInClientOptionParser.java index 227e498510..41369cee84 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/helpers/SplitsInClientOptionParser.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/helpers/SplitsInClientOptionParser.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -68,5 +68,4 @@ public boolean parse(String[] args, boolean defaultVal) throws ParseException { otherArgs = cmdLine.getArgs(); return defaultVal; } - } diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/processor/FilterByWordInputProcessor.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/processor/FilterByWordInputProcessor.java index 15d6e82d75..c0e5c75fab 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/processor/FilterByWordInputProcessor.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/processor/FilterByWordInputProcessor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -56,7 +56,6 @@ public FilterByWordInputProcessor(ProcessorContext context) { super(context); } - @Override public void initialize() throws Exception { Configuration conf = TezUtils.createConfFromUserPayload(getContext().getUserPayload()); @@ -69,7 +68,6 @@ public void initialize() throws Exception { @Override public void handleEvents(List processorEvents) { throw new UnsupportedOperationException("Not expecting any events to the broadcast processor"); - } @Override @@ -81,10 +79,10 @@ public void close() throws Exception { @Override public void run(Map _inputs, - Map _outputs) throws Exception { + Map _outputs) throws Exception { this.inputs = _inputs; this.outputs = _outputs; - this.progressHelper = new ProgressHelper(this.inputs, getContext(),this.getClass().getSimpleName()); + this.progressHelper = new ProgressHelper(this.inputs, getContext(), this.getClass().getSimpleName()); if (_inputs.size() != 1) { throw new IllegalStateException("FilterByWordInputProcessor processor can only work with a single input"); } @@ -92,7 +90,7 @@ public void run(Map _inputs, if (_outputs.size() != 1) { throw new IllegalStateException("FilterByWordInputProcessor processor can only work with a single output"); } - + for (LogicalInput input : _inputs.values()) { input.start(); } @@ -101,12 +99,12 @@ public void run(Map _inputs, } LogicalInput li = _inputs.values().iterator().next(); - if (! (li instanceof MRInput)) { + if (!(li instanceof MRInput)) { throw new IllegalStateException("FilterByWordInputProcessor processor can only work with MRInput"); } LogicalOutput lo = _outputs.values().iterator().next(); - if (! (lo instanceof UnorderedKVOutput)) { + if (!(lo instanceof UnorderedKVOutput)) { throw new IllegalStateException("FilterByWordInputProcessor processor can only work with OnFileUnorderedKVOutput"); } progressHelper.scheduleProgressTaskService(0, 100); diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/processor/FilterByWordOutputProcessor.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/processor/FilterByWordOutputProcessor.java index 7acaf7e0ac..97b08e11b5 100644 --- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/processor/FilterByWordOutputProcessor.java +++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/processor/FilterByWordOutputProcessor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,7 +34,6 @@ import org.apache.tez.runtime.library.api.KeyValueWriter; import org.apache.tez.runtime.library.input.UnorderedKVInput; - public class FilterByWordOutputProcessor extends SimpleMRProcessor { private static final Logger LOG = LoggerFactory.getLogger(MapProcessor.class); @@ -43,7 +42,6 @@ public FilterByWordOutputProcessor(ProcessorContext context) { super(context); } - @Override public void handleEvents(List processorEvents) { throw new UnsupportedOperationException("Not expecting any events to the broadcast output processor"); @@ -57,7 +55,7 @@ public void close() throws Exception { @Override public void run() throws Exception { - + if (inputs.size() != 1) { throw new IllegalStateException("FilterByWordOutputProcessor processor can only work with a single input"); } @@ -74,12 +72,12 @@ public void run() throws Exception { } LogicalInput li = inputs.values().iterator().next(); - if (! (li instanceof UnorderedKVInput)) { + if (!(li instanceof UnorderedKVInput)) { throw new IllegalStateException("FilterByWordOutputProcessor processor can only work with ShuffledUnorderedKVInput"); } LogicalOutput lo = outputs.values().iterator().next(); - if (! (lo instanceof MROutput)) { + if (!(lo instanceof MROutput)) { throw new IllegalStateException("FilterByWordOutputProcessor processor can only work with MROutput"); } diff --git a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobs.java b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobs.java index c00ea36cbc..2d07d31fe8 100644 --- a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobs.java +++ b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobs.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.mapreduce; @@ -68,7 +68,7 @@ public static void setup() throws IOException { conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1); conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR); dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2) - .format(true).racks(null).build(); + .format(true).racks(null).build(); remoteFs = dfsCluster.getFileSystem(); } catch (IOException io) { throw new RuntimeException("problem starting mini dfs cluster", io); @@ -76,7 +76,7 @@ public static void setup() throws IOException { if (!(new File(MiniTezCluster.APPJAR)).exists()) { LOG.info("MRAppJar " + MiniTezCluster.APPJAR - + " not found. Not running test."); + + " not found. Not running test."); return; } @@ -91,7 +91,6 @@ public static void setup() throws IOException { mrrTezCluster.init(conf); mrrTezCluster.start(); } - } @AfterClass @@ -106,14 +105,14 @@ public static void tearDown() { } } - @Test (timeout = 60000) + @Test(timeout = 60000) public void testMRRSleepJob() throws IOException, InterruptedException, ClassNotFoundException { LOG.info("\n\n\nStarting testMRRSleepJob()."); if (!(new File(MiniTezCluster.APPJAR)).exists()) { LOG.info("MRAppJar " + MiniTezCluster.APPJAR - + " not found. Not running test."); + + " not found. Not running test."); return; } @@ -138,22 +137,22 @@ public void testMRRSleepJob() throws IOException, InterruptedException, // to http://localhost:53419/proxy/application_1430963524753_0005/ui/ // So here use String#contains to verify. Assert.assertTrue("Tracking URL was " + trackingUrl + - " but didn't Match Job ID " + jobId , - trackingUrl.contains(jobId.substring(jobId.indexOf("_")))); + " but didn't Match Job ID " + jobId, + trackingUrl.contains(jobId.substring(jobId.indexOf("_")))); // FIXME once counters and task progress can be obtained properly // TODO use dag client to test counters and task progress? // what about completed jobs? } - @Test (timeout = 60000) + @Test(timeout = 60000) public void testRandomWriter() throws IOException, InterruptedException, ClassNotFoundException { LOG.info("\n\n\nStarting testRandomWriter()."); if (!(new File(MiniTezCluster.APPJAR)).exists()) { LOG.info("MRAppJar " + MiniTezCluster.APPJAR - + " not found. Not running test."); + + " not found. Not running test."); return; } @@ -173,8 +172,8 @@ public void testRandomWriter() throws IOException, InterruptedException, Assert.assertTrue(succeeded); Assert.assertEquals(JobStatus.State.SUCCEEDED, job.getJobState()); Assert.assertTrue("Tracking URL was " + trackingUrl + - " but didn't Match Job ID " + jobId , - trackingUrl.contains(jobId.substring(jobId.indexOf("_")))); + " but didn't Match Job ID " + jobId, + trackingUrl.contains(jobId.substring(jobId.indexOf("_")))); // Make sure there are three files in the output-dir @@ -190,11 +189,9 @@ public void testRandomWriter() throws IOException, InterruptedException, } } Assert.assertEquals("Number of part files is wrong!", 3, count); - } - - @Test (timeout = 60000) + @Test(timeout = 60000) public void testFailingJob() throws IOException, InterruptedException, ClassNotFoundException { @@ -202,7 +199,7 @@ public void testFailingJob() throws IOException, InterruptedException, if (!(new File(MiniTezCluster.APPJAR)).exists()) { LOG.info("MRAppJar " + MiniTezCluster.APPJAR - + " not found. Not running test."); + + " not found. Not running test."); return; } @@ -228,7 +225,7 @@ public void testFailingJob() throws IOException, InterruptedException, // TODO verify failed task diagnostics } - @Test (timeout = 60000) + @Test(timeout = 60000) public void testFailingAttempt() throws IOException, InterruptedException, ClassNotFoundException { @@ -236,7 +233,7 @@ public void testFailingAttempt() throws IOException, InterruptedException, if (!(new File(MiniTezCluster.APPJAR)).exists()) { LOG.info("MRAppJar " + MiniTezCluster.APPJAR - + " not found. Not running test."); + + " not found. Not running test."); return; } @@ -262,14 +259,14 @@ public void testFailingAttempt() throws IOException, InterruptedException, // TODO verify failed task diagnostics } - @Test (timeout = 60000) + @Test(timeout = 60000) public void testMRRSleepJobWithCompression() throws IOException, InterruptedException, ClassNotFoundException { LOG.info("\n\n\nStarting testMRRSleepJobWithCompression()."); if (!(new File(MiniTezCluster.APPJAR)).exists()) { LOG.info("MRAppJar " + MiniTezCluster.APPJAR - + " not found. Not running test."); + + " not found. Not running test."); return; } @@ -296,8 +293,8 @@ public void testMRRSleepJobWithCompression() throws IOException, Assert.assertTrue(succeeded); Assert.assertEquals(JobStatus.State.SUCCEEDED, job.getJobState()); Assert.assertTrue("Tracking URL was " + trackingUrl + - " but didn't Match Job ID " + jobId , - trackingUrl.contains(jobId.substring(jobId.indexOf("_")))); + " but didn't Match Job ID " + jobId, + trackingUrl.contains(jobId.substring(jobId.indexOf("_")))); // FIXME once counters and task progress can be obtained properly // TODO use dag client to test counters and task progress? @@ -361,5 +358,4 @@ public Void run() throws Exception { // TODO later: add explicit "isUber()" checks of some sort } */ - } diff --git a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java index 95d5bcf30b..7d01005a20 100644 --- a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java +++ b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -158,7 +158,7 @@ public static void setup() throws IOException { } catch (IOException io) { throw new RuntimeException("problem starting mini dfs cluster", io); } - + if (mrrTezCluster == null) { mrrTezCluster = new MiniTezCluster(TestMRRJobsDAGApi.class.getName(), 1, 1, 1); @@ -169,7 +169,6 @@ public static void setup() throws IOException { mrrTezCluster.init(conf); mrrTezCluster.start(); } - } @AfterClass @@ -184,7 +183,7 @@ public static void tearDown() { } // TODO Add cleanup code. } - + @Test(timeout = 60000) public void testSleepJob() throws TezException, IOException, InterruptedException { SleepProcessorConfig spConf = new SleepProcessorConfig(1); @@ -224,7 +223,7 @@ public void testSleepJob() throws TezException, IOException, InterruptedExceptio assertTrue("Memory used by AM is supposed to be >0", dagStatus.getMemoryUsedByAM() > 0); assertTrue("Memory used by tasks is supposed to be >0", dagStatus.getMemoryUsedByTasks() > 0); - ExampleDriver.printDAGStatus(dagClient, new String[] { "SleepVertex" }, true, true); + ExampleDriver.printDAGStatus(dagClient, new String[]{"SleepVertex"}, true, true); tezSession.stop(); } @@ -264,7 +263,7 @@ public void testNonDefaultFSStagingDir() throws Exception { assertNotNull(dagStatus.getDAGCounters()); assertNotNull(dagStatus.getDAGCounters().getGroup(FileSystemCounter.class.getName())); assertNotNull(dagStatus.getDAGCounters().findCounter(TaskCounter.GC_TIME_MILLIS)); - ExampleDriver.printDAGStatus(dagClient, new String[] { "SleepVertex" }, true, true); + ExampleDriver.printDAGStatus(dagClient, new String[]{"SleepVertex"}, true, true); tezSession.stop(); } @@ -328,7 +327,7 @@ public void testHistoryLogging() throws IOException, // client. @Test(timeout = 60000) public void testMRRSleepJobDagSubmit() throws IOException, - InterruptedException, TezException, ClassNotFoundException, YarnException { + InterruptedException, TezException, ClassNotFoundException, YarnException { State finalState = testMRRSleepJobDagSubmitCore(false, false, false, false); Assert.assertEquals(DAGStatus.State.SUCCEEDED, finalState); @@ -339,7 +338,7 @@ public void testMRRSleepJobDagSubmit() throws IOException, // Submits a simple 5 stage sleep job using the DAG submit API. Then kills it. @Test(timeout = 60000) public void testMRRSleepJobDagSubmitAndKill() throws IOException, - InterruptedException, TezException, ClassNotFoundException, YarnException { + InterruptedException, TezException, ClassNotFoundException, YarnException { State finalState = testMRRSleepJobDagSubmitCore(false, true, false, false); Assert.assertEquals(DAGStatus.State.KILLED, finalState); @@ -350,7 +349,7 @@ public void testMRRSleepJobDagSubmitAndKill() throws IOException, // Submits a DAG to AM via RPC after AM has started @Test(timeout = 60000) public void testMRRSleepJobViaSession() throws IOException, - InterruptedException, TezException, ClassNotFoundException, YarnException { + InterruptedException, TezException, ClassNotFoundException, YarnException { State finalState = testMRRSleepJobDagSubmitCore(true, false, false, false); Assert.assertEquals(DAGStatus.State.SUCCEEDED, finalState); @@ -417,9 +416,9 @@ private void stopAndVerifyYarnApp(TezClient tezSession) throws TezException, if (appReport.getYarnApplicationState().equals( YarnApplicationState.FINISHED) || appReport.getYarnApplicationState().equals( - YarnApplicationState.FAILED) + YarnApplicationState.FAILED) || appReport.getYarnApplicationState().equals( - YarnApplicationState.KILLED)) { + YarnApplicationState.KILLED)) { break; } } @@ -431,7 +430,6 @@ private void stopAndVerifyYarnApp(TezClient tezSession) throws TezException, appReport.getFinalApplicationStatus()); } - @Test(timeout = 120000) public void testAMRelocalizationConflict() throws Exception { Path relocPath = new Path("/tmp/relocalizationfilefound"); @@ -458,7 +456,7 @@ public void testAMRelocalizationConflict() throws Exception { try { testMRRSleepJobDagSubmitCore(true, false, false, - tezSession, true, MRInputAMSplitGeneratorRelocalizationTest.class, additionalResources); + tezSession, true, MRInputAMSplitGeneratorRelocalizationTest.class, additionalResources); Assert.fail("should have failed"); } catch (Exception ex) { // expected @@ -488,7 +486,7 @@ private TezClient createTezSession() throws IOException, TezException { // Submits a DAG to AM via RPC after AM has started @Test(timeout = 120000) public void testMultipleMRRSleepJobViaSession() throws IOException, - InterruptedException, TezException, ClassNotFoundException, YarnException { + InterruptedException, TezException, ClassNotFoundException, YarnException { Path remoteStagingDir = remoteFs.makeQualified(new Path("/tmp", String .valueOf(new Random().nextInt(100000)))); remoteFs.mkdirs(remoteStagingDir); @@ -519,7 +517,7 @@ public void testMultipleMRRSleepJobViaSession() throws IOException, // Submits a simple 5 stage sleep job using tez session. Then kills it. @Test(timeout = 60000) public void testMRRSleepJobDagSubmitAndKillViaRPC() throws IOException, - InterruptedException, TezException, ClassNotFoundException, YarnException { + InterruptedException, TezException, ClassNotFoundException, YarnException { State finalState = testMRRSleepJobDagSubmitCore(true, true, false, false); Assert.assertEquals(DAGStatus.State.KILLED, finalState); @@ -530,7 +528,7 @@ public void testMRRSleepJobDagSubmitAndKillViaRPC() throws IOException, // Create and close a tez session without submitting a job @Test(timeout = 60000) public void testTezSessionShutdown() throws IOException, - InterruptedException, TezException, ClassNotFoundException, YarnException { + InterruptedException, TezException, ClassNotFoundException, YarnException { testMRRSleepJobDagSubmitCore(true, false, true, false); } @@ -615,7 +613,7 @@ public State testMRRSleepJobDagSubmitCore( UserPayload stage1Payload = TezUtils.createUserPayloadFromConf(stage1Conf); UserPayload stage2Payload = TezUtils.createUserPayloadFromConf(stage2Conf); UserPayload stage3Payload = TezUtils.createUserPayloadFromConf(stage3Conf); - + DAG dag = DAG.create("testMRRSleepJobDagSubmit-" + random.nextInt(1000)); Class inputInitializerClazz = @@ -624,7 +622,6 @@ public State testMRRSleepJobDagSubmitCore( : null; LOG.info("Using initializer class: " + initializerClass); - DataSourceDescriptor dsd; if (!genSplitsInAM) { dsd = MRInputHelpers @@ -680,7 +677,7 @@ public State testMRRSleepJobDagSubmitCore( dag.addEdge(edge2); TezConfiguration tezConf = new TezConfiguration( - mrrTezCluster.getConfig()); + mrrTezCluster.getConfig()); tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString()); @@ -702,7 +699,7 @@ public State testMRRSleepJobDagSubmitCore( } else { tezSession = reUseTezSession; } - if(!dagViaRPC) { + if (!dagViaRPC) { // TODO Use utility method post TEZ-205 to figure out AM arguments etc. dagClient = tezSession.submitDAG(dag); } @@ -712,7 +709,7 @@ public State testMRRSleepJobDagSubmitCore( yarnClient.init(mrrTezCluster.getConfig()); yarnClient.start(); boolean sentKillSession = false; - while(true) { + while (true) { Thread.sleep(500l); ApplicationReport appReport = yarnClient.getApplicationReport(tezSession.getAppMasterApplicationId()); @@ -744,7 +741,7 @@ public State testMRRSleepJobDagSubmitCore( return null; } - if(dagViaRPC) { + if (dagViaRPC) { LOG.info("Submitting dag to tez session with appId=" + tezSession.getAppMasterApplicationId() + " and Dag Name=" + dag.getName()); if (additionalLocalResources != null) { @@ -759,7 +756,7 @@ public State testMRRSleepJobDagSubmitCore( LOG.info("Waiting for job to complete. Sleeping for 500ms." + " Current state: " + dagStatus.getState()); Thread.sleep(500l); - if(killDagWhileRunning + if (killDagWhileRunning && dagStatus.getState() == DAGStatus.State.RUNNING) { LOG.info("Killing running dag/session"); if (dagViaRPC) { @@ -777,7 +774,7 @@ public State testMRRSleepJobDagSubmitCore( } private static LocalResource createLocalResource(FileSystem fc, Path file, - LocalResourceType type, LocalResourceVisibility visibility) + LocalResourceType type, LocalResourceVisibility visibility) throws IOException { FileStatus fstat = fc.getFileStatus(file); URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat @@ -788,7 +785,7 @@ private static LocalResource createLocalResource(FileSystem fc, Path file, return LocalResource.newInstance(resourceURL, type, visibility, resourceSize, resourceModificationTime); } - + @Test(timeout = 60000) public void testVertexGroups() throws Exception { LOG.info("Running Group Test"); @@ -802,7 +799,7 @@ public void testVertexGroups() throws Exception { writer.write("efgh "); writer.close(); out.close(); - + UnionExample job = new UnionExample(); if (job.run(inPath.toString(), outPath.toString(), mrrTezCluster.getConfig())) { LOG.info("Success VertexGroups Test"); @@ -810,7 +807,7 @@ public void testVertexGroups() throws Exception { throw new TezUncheckedException("VertexGroups Test Failed"); } } - + @Test(timeout = 60000) public void testBroadcastAndOneToOne() throws Exception { LOG.info("Running BroadcastAndOneToOne Test"); @@ -825,6 +822,7 @@ public void testBroadcastAndOneToOne() throws Exception { // This class should not be used by more than one test in a single run, since // the path it writes to is not dynamic. private static String RELOCALIZATION_TEST_CLASS_NAME = "AMClassloadTestDummyClass"; + public static class MRInputAMSplitGeneratorRelocalizationTest extends MRInputAMSplitGenerator { public MRInputAMSplitGeneratorRelocalizationTest( @@ -833,7 +831,7 @@ public MRInputAMSplitGeneratorRelocalizationTest( } @Override - public List initialize() throws Exception { + public List initialize() throws Exception { MRInputUserPayloadProto userPayloadProto = MRInputHelpers .parseMRInputPayload(getContext().getInputUserPayload()); Configuration conf = TezUtils.createConfFromByteString(userPayloadProto @@ -852,7 +850,7 @@ public List initialize() throws Exception { return super.initialize(); } } - + private static void createTestJar(OutputStream outStream, String dummyClassName) throws URISyntaxException, IOException { JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); @@ -887,6 +885,7 @@ private static void createTestJar(OutputStream outStream, String dummyClassName) private static class SimpleJavaFileObjectImpl extends SimpleJavaFileObject { static final String code = "public class AMClassloadTestDummyClass {}"; + SimpleJavaFileObjectImpl(URI uri, Kind kind) { super(uri, kind); } diff --git a/tez-tests/src/test/java/org/apache/tez/test/AMShutdownController.java b/tez-tests/src/test/java/org/apache/tez/test/AMShutdownController.java index 4baf6de98d..c12a49494e 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/AMShutdownController.java +++ b/tez-tests/src/test/java/org/apache/tez/test/AMShutdownController.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -27,10 +27,10 @@ public abstract class AMShutdownController { private List historyEvents = new ArrayList(); - + protected AppContext appContext; protected RecoveryService recoveryService; - + public AMShutdownController(AppContext appContext, RecoveryService recoveryService) { this.appContext = appContext; this.recoveryService = recoveryService; @@ -50,8 +50,8 @@ public void postHandleHistoryEvent(DAGHistoryEvent event) { } protected abstract boolean shouldShutdownPreEvent(DAGHistoryEvent curEvent, - List historyEvents); + List historyEvents); protected abstract boolean shouldShutdownPostEvent(DAGHistoryEvent curEvent, - List historyEvents); + List historyEvents); } diff --git a/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java b/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java index 9305711744..166bdc5ac5 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java +++ b/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -32,6 +32,7 @@ import org.apache.tez.dag.api.client.DAGClient; import org.apache.tez.dag.api.client.DAGStatus; import org.apache.hadoop.util.GenericOptionsParser; + /** * Run a DAG on a cluster with the given configuration. Starts a TezSession * using default cluster configuration from installation. Then uses reflection @@ -42,13 +43,13 @@ * then executed in the session. Returns success if DAG succeeds. */ public class FaultToleranceTestRunner { - + static String DEFAULT_FT_STAGING_DIR = "tmp"; static String FT_STAGING_DIR = "tez.test-fault-tolerance.staging-dir"; Configuration conf = null; TezClient tezSession = null; Resource defaultResource = Resource.newInstance(100, 0); - + void setup() throws Exception { TezConfiguration tezConf = null; if (conf == null) { @@ -69,44 +70,44 @@ void setup() throws Exception { Path remoteStagingDir = defaultFs.makeQualified(new Path(testRootDir, String .valueOf(new Random().nextInt(100000)))); TezClientUtils.ensureStagingDirExists(tezConf, remoteStagingDir); - + tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString()); tezSession = TezClient.create("FaultToleranceTestRunner", tezConf); tezSession.start(); } - + void tearDown() throws Exception { if (tezSession != null) { tezSession.stop(); } } - + DAG getDAG(String className, String confFilePath) throws Exception { Class clazz = Class.forName(className); Method method = clazz.getMethod("createDAG", Configuration.class); - + Configuration testConf = new Configuration(false); if (confFilePath != null) { Path confPath = new Path(confFilePath); testConf.addResource(confPath); } - + DAG dag = (DAG) method.invoke(null, testConf); - + return dag; } - + boolean run(Configuration conf, String className, String confFilePath) throws Exception { this.conf = conf; setup(); - + try { tezSession.waitTillReady(); - + DAG dag = getDAG(className, confFilePath); - + DAGClient dagClient = tezSession.submitDAG(dag); DAGStatus dagStatus = dagClient.getDAGStatus(null); while (!dagStatus.isCompleted()) { @@ -117,24 +118,23 @@ boolean run(Configuration conf, String className, String confFilePath) throws Ex Thread.sleep(500); dagStatus = dagClient.getDAGStatus(null); } - + if (dagStatus.getState() == DAGStatus.State.SUCCEEDED) { return true; } - } finally { tearDown(); } - + return false; } - + static void printUsage() { System.err.println( "Usage: " + " FaultToleranceTestRunner [generic options] "); GenericOptionsParser.printGenericCommandUsage(System.err); } - + public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); @@ -149,13 +149,13 @@ public static void main(String[] args) throws Exception { printUsage(); System.exit(1); } - + FaultToleranceTestRunner job = new FaultToleranceTestRunner(); if (job.run(conf, className, confFilePath)) { System.out.println("Succeeded."); } else { System.out.println("Failed."); System.exit(2); - } + } } } diff --git a/tez-tests/src/test/java/org/apache/tez/test/MiniTezCluster.java b/tez-tests/src/test/java/org/apache/tez/test/MiniTezCluster.java index 9af1e604b2..a722973b6e 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/MiniTezCluster.java +++ b/tez-tests/src/test/java/org/apache/tez/test/MiniTezCluster.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.test; @@ -85,7 +85,7 @@ public MiniTezCluster(String testName, int noOfNMs) { } public MiniTezCluster(String testName, int noOfNMs, - int numLocalDirs, int numLogDirs) { + int numLocalDirs, int numLogDirs) { super(testName, noOfNMs, numLocalDirs, numLogDirs); } @@ -142,12 +142,12 @@ public void serviceInit(Configuration conf) throws Exception { conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false); conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false); - conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000"); + conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000"); conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 1); conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_TIMEOUT_KEY, 1000); - conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,0); + conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0); conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0); - conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY,1000); + conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY, 1000); try { Path stagingPath = FileContext.getFileContext(conf).makeQualified( @@ -167,7 +167,7 @@ public void serviceInit(Configuration conf) throws Exception { .getAbsolutePath()); } } - FileContext fc=FileContext.getFileContext(stagingPath.toUri(), conf); + FileContext fc = FileContext.getFileContext(stagingPath.toUri(), conf); if (fc.util().exists(stagingPath)) { LOG.info(stagingPath + " exists! deleting..."); fc.delete(stagingPath, true); @@ -190,7 +190,7 @@ public void serviceInit(Configuration conf) throws Exception { conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[]{ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID}); conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, - ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID), ShuffleHandler.class, + ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID), ShuffleHandler.class, Service.class); // Non-standard shuffle port conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0); @@ -251,7 +251,7 @@ private void waitForAppsToFinish() { @Override public boolean apply(ApplicationReport appReport) { return EnumSet.of(YarnApplicationState.NEW, YarnApplicationState.NEW_SAVING, - YarnApplicationState.SUBMITTED, YarnApplicationState.ACCEPTED, YarnApplicationState.RUNNING) + YarnApplicationState.SUBMITTED, YarnApplicationState.ACCEPTED, YarnApplicationState.RUNNING) .contains(appReport.getYarnApplicationState()); } }); @@ -263,7 +263,6 @@ public boolean apply(ApplicationReport appReport) { Thread.sleep(1000); } while (waitEndTime != -1 && waitEndTime > System.currentTimeMillis()); - if (unCompletedApps != null && !unCompletedApps.isEmpty()) { LOG.info("Killing incomplete applications in MiniTezCluster" + ", incompleteAppsCount=" + unCompletedApps.size()); @@ -302,7 +301,6 @@ public boolean apply(ApplicationReport appReport) { break; } } while (waitEndTime != -1 && waitEndTime > System.currentTimeMillis()); - } } catch (Exception e) { e.printStackTrace(); @@ -310,9 +308,8 @@ public boolean apply(ApplicationReport appReport) { yarnClient.stop(); } } - + public Path getConfigFilePath() { return confFilePath; } - } diff --git a/tez-tests/src/test/java/org/apache/tez/test/RecoveryServiceWithEventHandlingHook.java b/tez-tests/src/test/java/org/apache/tez/test/RecoveryServiceWithEventHandlingHook.java index 186ab7e659..89e9784eda 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/RecoveryServiceWithEventHandlingHook.java +++ b/tez-tests/src/test/java/org/apache/tez/test/RecoveryServiceWithEventHandlingHook.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -62,6 +62,7 @@ public class RecoveryServiceWithEventHandlingHook extends RecoveryService { private static final Logger LOG = LoggerFactory.getLogger(RecoveryServiceWithEventHandlingHook.class); private RecoveryServiceHook hook; private boolean shutdownInvoked = false; + public RecoveryServiceWithEventHandlingHook(AppContext appContext) { super(appContext); } @@ -71,7 +72,7 @@ public void serviceInit(Configuration conf) throws Exception { super.serviceInit(conf); String clazz = conf.get(AM_RECOVERY_SERVICE_HOOK_CLASS); Preconditions.checkArgument(clazz != null, "RecoveryServiceHook class is not specified"); - this.hook = ReflectionUtils.createClazzInstance(clazz, + this.hook = ReflectionUtils.createClazzInstance(clazz, new Class[]{RecoveryServiceWithEventHandlingHook.class, AppContext.class}, new Object[]{this, super.appContext}); } @@ -88,7 +89,7 @@ protected void handleRecoveryEvent(DAGHistoryEvent event) throws IOException { @Override protected void handleSummaryEvent(TezDAGID dagID, HistoryEventType eventType, - SummaryEvent summaryEvent) throws IOException { + SummaryEvent summaryEvent) throws IOException { hook.preHandleSummaryEvent(eventType, summaryEvent); if (shutdownInvoked) { return; @@ -133,11 +134,10 @@ public RecoveryServiceHook(RecoveryServiceWithEventHandlingHook recoveryService, public abstract void postHandleRecoveryEvent(DAGHistoryEvent event) throws IOException; public abstract void preHandleSummaryEvent(HistoryEventType eventType, - SummaryEvent summaryEvent) throws IOException; + SummaryEvent summaryEvent) throws IOException; public abstract void postHandleSummaryEvent(HistoryEventType eventType, - SummaryEvent summaryEvent) throws IOException; - + SummaryEvent summaryEvent) throws IOException; } /** @@ -177,26 +177,25 @@ public void preHandleRecoveryEvent(DAGHistoryEvent event) public void postHandleRecoveryEvent(DAGHistoryEvent event) throws IOException { if (shutdownCondition.timing.equals(TIMING.POST) - && appContext.getApplicationAttemptId().getAttemptId() == 1 - && shutdownCondition.match(event.getHistoryEvent())) { + && appContext.getApplicationAttemptId().getAttemptId() == 1 + && shutdownCondition.match(event.getHistoryEvent())) { recoveryService.shutdown(); } } - + @Override public void preHandleSummaryEvent(HistoryEventType eventType, - SummaryEvent summaryEvent) throws IOException { + SummaryEvent summaryEvent) throws IOException { } @Override public void postHandleSummaryEvent(HistoryEventType eventType, - SummaryEvent summaryEvent) throws IOException { + SummaryEvent summaryEvent) throws IOException { } - } /** - * + * * Shutdown AM based on one recovery event if it is matched. * This would be serialized as property of TezConfiguration and deserialized at runtime. */ @@ -267,119 +266,119 @@ public TIMING getTiming() { public boolean match(HistoryEvent incomingEvent) { switch (event.getEventType()) { - case DAG_SUBMITTED: - if (incomingEvent.getEventType() == HistoryEventType.DAG_SUBMITTED) { - // only compare eventType - return true; - } - break; - - case DAG_INITIALIZED: - if (incomingEvent.getEventType() == HistoryEventType.DAG_INITIALIZED) { - // only compare eventType - return true; - } - break; - - case DAG_STARTED: - if (incomingEvent.getEventType() == HistoryEventType.DAG_STARTED) { - // only compare eventType - return true; - } - break; - - case DAG_FINISHED: - if (incomingEvent.getEventType() == HistoryEventType.DAG_FINISHED) { - // only compare eventType - return true; - } - break; - - case VERTEX_INITIALIZED: - if (incomingEvent.getEventType() == HistoryEventType.VERTEX_INITIALIZED) { - VertexInitializedEvent otherEvent = (VertexInitializedEvent) incomingEvent; - VertexInitializedEvent conditionEvent = (VertexInitializedEvent) event; - // compare vertexId; - return otherEvent.getVertexID().getId() == conditionEvent.getVertexID().getId(); - } - break; - - case VERTEX_STARTED: - if (incomingEvent.getEventType() == HistoryEventType.VERTEX_STARTED) { - VertexStartedEvent otherEvent = (VertexStartedEvent) incomingEvent; - VertexStartedEvent conditionEvent = (VertexStartedEvent) event; - // compare vertexId - return otherEvent.getVertexID().getId() == conditionEvent.getVertexID().getId(); - } - break; - - case VERTEX_FINISHED: - if (incomingEvent.getEventType() == HistoryEventType.VERTEX_FINISHED) { - VertexFinishedEvent otherEvent = (VertexFinishedEvent) incomingEvent; - VertexFinishedEvent conditionEvent = (VertexFinishedEvent) event; - // compare vertexId - return otherEvent.getVertexID().getId() == conditionEvent.getVertexID().getId(); - } - break; - case VERTEX_CONFIGURE_DONE: - if (incomingEvent.getEventType() == HistoryEventType.VERTEX_CONFIGURE_DONE) { - VertexConfigurationDoneEvent otherEvent = (VertexConfigurationDoneEvent) incomingEvent; - VertexConfigurationDoneEvent conditionEvent = (VertexConfigurationDoneEvent) event; - // compare vertexId - return otherEvent.getVertexID().getId() == conditionEvent.getVertexID().getId(); - } - break; - case TASK_STARTED: - if (incomingEvent.getEventType() == HistoryEventType.TASK_STARTED) { - TaskStartedEvent otherEvent = (TaskStartedEvent) incomingEvent; - TaskStartedEvent conditionEvent = (TaskStartedEvent) event; - // compare vertexId and taskId - return otherEvent.getVertexID().getId() == conditionEvent.getVertexID().getId() - && otherEvent.getTaskID().getId() == conditionEvent.getTaskID().getId(); - } - break; - - case TASK_FINISHED: - if (incomingEvent.getEventType() == HistoryEventType.TASK_FINISHED) { - TaskFinishedEvent otherEvent = (TaskFinishedEvent) incomingEvent; - TaskFinishedEvent conditionEvent = (TaskFinishedEvent) event; - // compare vertexId and taskId - return otherEvent.getVertexID().getId() == conditionEvent.getVertexID().getId() - && otherEvent.getTaskID().getId() == conditionEvent.getTaskID().getId(); - } - break; - - case TASK_ATTEMPT_STARTED: - if (incomingEvent.getEventType() == HistoryEventType.TASK_ATTEMPT_STARTED) { - TaskAttemptStartedEvent otherEvent = (TaskAttemptStartedEvent) incomingEvent; - TaskAttemptStartedEvent conditionEvent = (TaskAttemptStartedEvent) event; - // compare vertexId, taskId & taskAttemptId - return otherEvent.getVertexID().getId() - == conditionEvent.getVertexID().getId() - && otherEvent.getTaskID().getId() == conditionEvent.getTaskID().getId() - && otherEvent.getTaskAttemptID().getId() == conditionEvent.getTaskAttemptID().getId(); - } - break; - - case TASK_ATTEMPT_FINISHED: - if (incomingEvent.getEventType() == HistoryEventType.TASK_ATTEMPT_FINISHED) { - TaskAttemptFinishedEvent otherEvent = (TaskAttemptFinishedEvent) incomingEvent; - TaskAttemptFinishedEvent conditionEvent = (TaskAttemptFinishedEvent) event; - // compare vertexId, taskId & taskAttemptId - return otherEvent.getVertexID().getId() - == conditionEvent.getVertexID().getId() - && otherEvent.getTaskID().getId() == conditionEvent.getTaskID().getId() - && otherEvent.getTaskAttemptID().getId() == conditionEvent.getTaskAttemptID().getId(); - } - break; - default: - LOG.info("do nothing with event:" - + event.getEventType()); + case DAG_SUBMITTED: + if (incomingEvent.getEventType() == HistoryEventType.DAG_SUBMITTED) { + // only compare eventType + return true; + } + break; + + case DAG_INITIALIZED: + if (incomingEvent.getEventType() == HistoryEventType.DAG_INITIALIZED) { + // only compare eventType + return true; + } + break; + + case DAG_STARTED: + if (incomingEvent.getEventType() == HistoryEventType.DAG_STARTED) { + // only compare eventType + return true; + } + break; + + case DAG_FINISHED: + if (incomingEvent.getEventType() == HistoryEventType.DAG_FINISHED) { + // only compare eventType + return true; + } + break; + + case VERTEX_INITIALIZED: + if (incomingEvent.getEventType() == HistoryEventType.VERTEX_INITIALIZED) { + VertexInitializedEvent otherEvent = (VertexInitializedEvent) incomingEvent; + VertexInitializedEvent conditionEvent = (VertexInitializedEvent) event; + // compare vertexId; + return otherEvent.getVertexID().getId() == conditionEvent.getVertexID().getId(); + } + break; + + case VERTEX_STARTED: + if (incomingEvent.getEventType() == HistoryEventType.VERTEX_STARTED) { + VertexStartedEvent otherEvent = (VertexStartedEvent) incomingEvent; + VertexStartedEvent conditionEvent = (VertexStartedEvent) event; + // compare vertexId + return otherEvent.getVertexID().getId() == conditionEvent.getVertexID().getId(); + } + break; + + case VERTEX_FINISHED: + if (incomingEvent.getEventType() == HistoryEventType.VERTEX_FINISHED) { + VertexFinishedEvent otherEvent = (VertexFinishedEvent) incomingEvent; + VertexFinishedEvent conditionEvent = (VertexFinishedEvent) event; + // compare vertexId + return otherEvent.getVertexID().getId() == conditionEvent.getVertexID().getId(); + } + break; + case VERTEX_CONFIGURE_DONE: + if (incomingEvent.getEventType() == HistoryEventType.VERTEX_CONFIGURE_DONE) { + VertexConfigurationDoneEvent otherEvent = (VertexConfigurationDoneEvent) incomingEvent; + VertexConfigurationDoneEvent conditionEvent = (VertexConfigurationDoneEvent) event; + // compare vertexId + return otherEvent.getVertexID().getId() == conditionEvent.getVertexID().getId(); + } + break; + case TASK_STARTED: + if (incomingEvent.getEventType() == HistoryEventType.TASK_STARTED) { + TaskStartedEvent otherEvent = (TaskStartedEvent) incomingEvent; + TaskStartedEvent conditionEvent = (TaskStartedEvent) event; + // compare vertexId and taskId + return otherEvent.getVertexID().getId() == conditionEvent.getVertexID().getId() + && otherEvent.getTaskID().getId() == conditionEvent.getTaskID().getId(); + } + break; + + case TASK_FINISHED: + if (incomingEvent.getEventType() == HistoryEventType.TASK_FINISHED) { + TaskFinishedEvent otherEvent = (TaskFinishedEvent) incomingEvent; + TaskFinishedEvent conditionEvent = (TaskFinishedEvent) event; + // compare vertexId and taskId + return otherEvent.getVertexID().getId() == conditionEvent.getVertexID().getId() + && otherEvent.getTaskID().getId() == conditionEvent.getTaskID().getId(); + } + break; + + case TASK_ATTEMPT_STARTED: + if (incomingEvent.getEventType() == HistoryEventType.TASK_ATTEMPT_STARTED) { + TaskAttemptStartedEvent otherEvent = (TaskAttemptStartedEvent) incomingEvent; + TaskAttemptStartedEvent conditionEvent = (TaskAttemptStartedEvent) event; + // compare vertexId, taskId & taskAttemptId + return otherEvent.getVertexID().getId() + == conditionEvent.getVertexID().getId() + && otherEvent.getTaskID().getId() == conditionEvent.getTaskID().getId() + && otherEvent.getTaskAttemptID().getId() == conditionEvent.getTaskAttemptID().getId(); + } + break; + + case TASK_ATTEMPT_FINISHED: + if (incomingEvent.getEventType() == HistoryEventType.TASK_ATTEMPT_FINISHED) { + TaskAttemptFinishedEvent otherEvent = (TaskAttemptFinishedEvent) incomingEvent; + TaskAttemptFinishedEvent conditionEvent = (TaskAttemptFinishedEvent) event; + // compare vertexId, taskId & taskAttemptId + return otherEvent.getVertexID().getId() + == conditionEvent.getVertexID().getId() + && otherEvent.getTaskID().getId() == conditionEvent.getTaskID().getId() + && otherEvent.getTaskAttemptID().getId() == conditionEvent.getTaskAttemptID().getId(); + } + break; + default: + LOG.info("do nothing with event:" + + event.getEventType()); } return false; } - + public HistoryEventType getEventType() { return event.getEventType(); } @@ -396,7 +395,7 @@ public MultipleRoundRecoveryEventHook(RecoveryServiceWithEventHandlingHook recov this.shutdownCondition = new MultipleRoundShutdownCondition(); try { Preconditions.checkArgument(recoveryService.getConfig().get(MULTIPLE_ROUND_SHUTDOWN_CONDITION) != null, - MULTIPLE_ROUND_SHUTDOWN_CONDITION + " is not set in TezConfiguration"); + MULTIPLE_ROUND_SHUTDOWN_CONDITION + " is not set in TezConfiguration"); this.shutdownCondition.deserialize(recoveryService.getConfig().get(MULTIPLE_ROUND_SHUTDOWN_CONDITION)); } catch (IOException e) { throw new TezUncheckedException("Can not initialize MultipleRoundShutdownCondition", e); @@ -409,7 +408,7 @@ public void preHandleRecoveryEvent(DAGHistoryEvent event) throws IOException { if (attemptId <= shutdownCondition.size()) { SimpleShutdownCondition condition = shutdownCondition.getSimpleShutdownCondition(attemptId - 1); if (condition.timing.equals(TIMING.PRE) - && condition.match(event.getHistoryEvent())) { + && condition.match(event.getHistoryEvent())) { recoveryService.shutdown(); } } @@ -417,7 +416,7 @@ public void preHandleRecoveryEvent(DAGHistoryEvent event) throws IOException { @Override public void postHandleRecoveryEvent(DAGHistoryEvent event) throws IOException { - for (int i=0;i shutdownCond public String serialize() throws IOException { StringBuilder builder = new StringBuilder(); - for (int i=0; i< shutdownConditionList.size(); ++i) { + for (int i = 0; i < shutdownConditionList.size(); ++i) { builder.append(shutdownConditionList.get(i).serialize()); - if (i!=shutdownConditionList.size()-1) { + if (i != shutdownConditionList.size() - 1) { builder.append(";"); } } diff --git a/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG.java b/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG.java index 90a8cf61ad..48f736e2c0 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG.java +++ b/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -32,7 +32,7 @@ /** * Simple Test DAG with 2 vertices using TestProcessor/TestInput/TestOutput. - * + * * v1 * | * v2 @@ -43,9 +43,9 @@ public class SimpleTestDAG { public static String TEZ_SIMPLE_DAG_NUM_TASKS = "tez.simple-test-dag.num-tasks"; public static int TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT = 2; - - public static DAG createDAG(String name, - Configuration conf) throws Exception { + + public static DAG createDAG(String name, + Configuration conf) throws Exception { UserPayload payload = UserPayload.create(null); int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT; if (conf != null) { @@ -63,7 +63,7 @@ public static DAG createDAG(String name, TestInput.getInputDesc(payload)))); return dag; } - + public static DAG createDAG(Configuration conf) throws Exception { return createDAG("SimpleTestDAG", conf); } @@ -81,7 +81,7 @@ public static DAG createDAG(Configuration conf) throws Exception { * @return * @throws Exception */ - public static DAG createDAGForVertexOrder(String name, Configuration conf) throws Exception{ + public static DAG createDAGForVertexOrder(String name, Configuration conf) throws Exception { UserPayload payload = UserPayload.create(null); int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT; if (conf != null) { @@ -99,47 +99,47 @@ public static DAG createDAGForVertexOrder(String name, Configuration conf) throw // add vertex not in the topological order, since we are using this dag for testing vertex topological order dag.addVertex(v4) - .addVertex(v5) - .addVertex(v6) - .addVertex(v1) - .addVertex(v2) - .addVertex(v3) - .addEdge(Edge.create(v1, v3, - EdgeProperty.create(DataMovementType.SCATTER_GATHER, - DataSourceType.PERSISTED, - SchedulingType.SEQUENTIAL, - TestOutput.getOutputDesc(payload), - TestInput.getInputDesc(payload)))) - .addEdge(Edge.create(v2, v3, - EdgeProperty.create(DataMovementType.SCATTER_GATHER, - DataSourceType.PERSISTED, - SchedulingType.SEQUENTIAL, - TestOutput.getOutputDesc(payload), - TestInput.getInputDesc(payload)))) - .addEdge(Edge.create(v3, v4, - EdgeProperty.create(DataMovementType.SCATTER_GATHER, - DataSourceType.PERSISTED, - SchedulingType.SEQUENTIAL, - TestOutput.getOutputDesc(payload), - TestInput.getInputDesc(payload)))) - .addEdge(Edge.create(v3, v5, - EdgeProperty.create(DataMovementType.SCATTER_GATHER, - DataSourceType.PERSISTED, - SchedulingType.SEQUENTIAL, - TestOutput.getOutputDesc(payload), - TestInput.getInputDesc(payload)))) - .addEdge(Edge.create(v4, v6, - EdgeProperty.create(DataMovementType.SCATTER_GATHER, - DataSourceType.PERSISTED, - SchedulingType.SEQUENTIAL, - TestOutput.getOutputDesc(payload), - TestInput.getInputDesc(payload)))) - .addEdge(Edge.create(v5, v6, - EdgeProperty.create(DataMovementType.SCATTER_GATHER, - DataSourceType.PERSISTED, - SchedulingType.SEQUENTIAL, - TestOutput.getOutputDesc(payload), - TestInput.getInputDesc(payload)))); + .addVertex(v5) + .addVertex(v6) + .addVertex(v1) + .addVertex(v2) + .addVertex(v3) + .addEdge(Edge.create(v1, v3, + EdgeProperty.create(DataMovementType.SCATTER_GATHER, + DataSourceType.PERSISTED, + SchedulingType.SEQUENTIAL, + TestOutput.getOutputDesc(payload), + TestInput.getInputDesc(payload)))) + .addEdge(Edge.create(v2, v3, + EdgeProperty.create(DataMovementType.SCATTER_GATHER, + DataSourceType.PERSISTED, + SchedulingType.SEQUENTIAL, + TestOutput.getOutputDesc(payload), + TestInput.getInputDesc(payload)))) + .addEdge(Edge.create(v3, v4, + EdgeProperty.create(DataMovementType.SCATTER_GATHER, + DataSourceType.PERSISTED, + SchedulingType.SEQUENTIAL, + TestOutput.getOutputDesc(payload), + TestInput.getInputDesc(payload)))) + .addEdge(Edge.create(v3, v5, + EdgeProperty.create(DataMovementType.SCATTER_GATHER, + DataSourceType.PERSISTED, + SchedulingType.SEQUENTIAL, + TestOutput.getOutputDesc(payload), + TestInput.getInputDesc(payload)))) + .addEdge(Edge.create(v4, v6, + EdgeProperty.create(DataMovementType.SCATTER_GATHER, + DataSourceType.PERSISTED, + SchedulingType.SEQUENTIAL, + TestOutput.getOutputDesc(payload), + TestInput.getInputDesc(payload)))) + .addEdge(Edge.create(v5, v6, + EdgeProperty.create(DataMovementType.SCATTER_GATHER, + DataSourceType.PERSISTED, + SchedulingType.SEQUENTIAL, + TestOutput.getOutputDesc(payload), + TestInput.getInputDesc(payload)))); return dag; } diff --git a/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG3Vertices.java b/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG3Vertices.java index a48b2d696b..3fcf6643ca 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG3Vertices.java +++ b/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG3Vertices.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -32,7 +32,7 @@ /** * Simple Test DAG with 3 vertices using TestProcessor/TestInput/TestOutput. - * + * * v1 * | * v2 @@ -45,9 +45,9 @@ public class SimpleTestDAG3Vertices { public static String TEZ_SIMPLE_DAG_NUM_TASKS = "tez.simple-test-dag-3-vertices.num-tasks"; public static int TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT = 2; - - public static DAG createDAG(String name, - Configuration conf) throws Exception { + + public static DAG createDAG(String name, + Configuration conf) throws Exception { UserPayload payload = null; int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT; if (conf != null) { @@ -72,7 +72,7 @@ public static DAG createDAG(String name, TestInput.getInputDesc(payload)))); return dag; } - + public static DAG createDAG(Configuration conf) throws Exception { return createDAG("SimpleTestDAG3Vertices", conf); } diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestAM.java b/tez-tests/src/test/java/org/apache/tez/test/TestAM.java index 3e8a58cf7a..72eba7b198 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestAM.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestAM.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestAMRecovery.java b/tez-tests/src/test/java/org/apache/tez/test/TestAMRecovery.java index 7fe3b3acc1..91441df26e 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestAMRecovery.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestAMRecovery.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -176,9 +176,9 @@ public void setup() throws Exception { tezConf.setBoolean( RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, true); - tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,0); + tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0); tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0); - tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY,1000); + tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY, 1000); tezSession = TezClient.create("TestDAGRecovery", tezConf); tezSession.start(); } @@ -288,7 +288,6 @@ public void testVertexPartialFinished_One2One() throws Exception { // finished in attempt 2 assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 0).size()); assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 1).size()); - } /** @@ -320,7 +319,6 @@ public void testVertexCompletelyFinished_One2One() throws Exception { // finished in attempt 2 assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 0).size()); assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 1).size()); - } /** @@ -352,7 +350,6 @@ public void testVertexPartiallyFinished_ScatterGather() throws Exception { // finished in attempt 2 assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 0).size()); assertEquals(1, findTaskAttemptFinishedEvent(historyEvents2, 0, 1).size()); - } /** @@ -406,7 +403,6 @@ public void testHighMaxAttempt() throws Exception { createDAG("HighMaxAttempt", FailOnAttemptVertexManager.class, DataMovementType.SCATTER_GATHER, false); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); - } TezCounters runDAGAndVerify(DAG dag, DAGStatus.State finalState) throws Exception { @@ -431,7 +427,7 @@ TezCounters runDAGAndVerify(DAG dag, DAGStatus.State finalState) throws Exceptio * @throws IOException */ private DAG createDAG(String dagName, Class vertexManagerClass, DataMovementType dmType, - boolean failOnParitialCompleted) throws IOException { + boolean failOnParitialCompleted) throws IOException { if (failOnParitialCompleted) { tezConf.set(FAIL_ON_PARTIAL_FINISHED, "true"); } else { @@ -481,7 +477,7 @@ private List readRecoveryLog(int attemptNum) throws IOException { TezCommonUtils.getRecoveryPath(tezSystemStagingDir, tezConf); FileSystem fs = tezSystemStagingDir.getFileSystem(tezConf); List historyEvents = new ArrayList(); - for (int i=1; i <= attemptNum; ++i) { + for (int i = 1; i <= attemptNum; ++i) { Path currentAttemptRecoveryDataDir = TezCommonUtils.getAttemptRecoveryPath(recoveryDataDir, i); Path recoveryFilePath = @@ -498,7 +494,7 @@ private List readRecoveryLog(int attemptNum) throws IOException { private void printHistoryEvents(List historyEvents, int attemptId) { LOG.info("RecoveryLogs from attempt:" + attemptId); - for(HistoryEvent historyEvent : historyEvents) { + for (HistoryEvent historyEvent : historyEvents) { LOG.info("Parsed event from recovery stream" + ", eventType=" + historyEvent.getEventType() + ", event=" + historyEvent); @@ -530,7 +526,7 @@ public void initialize() { @Override public void onSourceTaskCompleted(TaskAttemptIdentifier attempt) { super.onSourceTaskCompleted(attempt); - completedTaskNum ++; + completedTaskNum++; if (getContext().getDAGAttemptNumber() == 1) { if (conf.getBoolean(FAIL_ON_PARTIAL_FINISHED, true)) { if (completedTaskNum == 1) { @@ -570,7 +566,7 @@ public void initialize() { @Override public void onSourceTaskCompleted(TaskAttemptIdentifier attempt) { super.onSourceTaskCompleted(attempt); - completedTaskNum ++; + completedTaskNum++; if (getContext().getDAGAttemptNumber() == 1) { if (conf.getBoolean(FAIL_ON_PARTIAL_FINISHED, true)) { if (completedTaskNum == 1) { @@ -611,7 +607,7 @@ public void initialize() { @Override public void onSourceTaskCompleted(TaskAttemptIdentifier attempt) { super.onSourceTaskCompleted(attempt); - completedTaskNum ++; + completedTaskNum++; if (getContext().getDAGAttemptNumber() == 1) { if (conf.getBoolean(FAIL_ON_PARTIAL_FINISHED, true)) { if (completedTaskNum == 1) { @@ -627,7 +623,6 @@ public void onSourceTaskCompleted(TaskAttemptIdentifier attempt) { } } - /** * VertexManager which control schedule only one task when it is test case of partially-finished. * @@ -657,13 +652,13 @@ public void onVertexStarted(List completions) // only schedule one task if it is partiallyFinished case if (conf.getBoolean(FAIL_ON_PARTIAL_FINISHED, true)) { getContext().scheduleTasks(Lists.newArrayList(ScheduleTaskRequest.create(0, null))); - return ; + return; } } // schedule all tasks when it is not partiallyFinished int taskNum = getContext().getVertexNumTasks(getContext().getVertexName()); List taskWithLocationHints = new ArrayList(); - for (int i=0;i completions) @Override public void onSourceTaskCompleted(TaskAttemptIdentifier attempt) throws Exception { - + } @Override public void onVertexManagerEventReceived(VertexManagerEvent vmEvent) throws Exception { - + } @Override public void onRootVertexInitialized(String inputName, - InputDescriptor inputDescriptor, List events) throws Exception { - + InputDescriptor inputDescriptor, List events) throws Exception { + } } @@ -763,5 +758,4 @@ public static ProcessorDescriptor getProcDesc() { return ProcessorDescriptor.create(DoNothingProcessor.class.getName()); } } - } diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java b/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java index cf4744b2a2..14d724b244 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -119,7 +119,7 @@ public static void afterClass() throws InterruptedException { } @Before - public void setup() throws Exception { + public void setup() throws Exception { LOG.info("Starting session"); Path remoteStagingDir = remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String .valueOf(new Random().nextInt(100000)))); @@ -136,9 +136,9 @@ public void setup() throws Exception { tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, " -Xmx256m"); tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true); tezConf.set(TezConfiguration.TEZ_AM_STAGING_SCRATCH_DATA_AUTO_DELETE, "false"); - tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,0); + tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0); tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0); - tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY,1000); + tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY, 1000); tezSession = TezClient.create("TestDAGRecovery", tezConf); tezSession.start(); @@ -173,19 +173,19 @@ void runDAGAndVerify(DAG dag, DAGStatus.State finalState) throws Exception { Assert.assertEquals(finalState, dagStatus.getState()); } - @Test(timeout=120000) + @Test(timeout = 120000) public void testBasicRecovery() throws Exception { DAG dag = MultiAttemptDAG.createDAG("TestBasicRecovery", null); // add input to v1 to make sure that there will be init events for v1 (TEZ-1345) DataSourceDescriptor dataSource = DataSourceDescriptor.create(InputDescriptor.create(NoOpInput.class.getName()), - InputInitializerDescriptor.create(TestRootInputInitializer.class.getName()), null); + InputInitializerDescriptor.create(TestRootInputInitializer.class.getName()), null); dag.getVertex("v1").addDataSource("Input", dataSource); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - @Test(timeout=120000) + @Test(timeout = 120000) public void testDelayedInit() throws Exception { DAG dag = SimpleVTestDAG.createDAG("DelayedInitDAG", null); dag.getVertex("v1").addDataSource( @@ -196,5 +196,4 @@ public void testDelayedInit() throws Exception { .getName()), null)); runDAGAndVerify(dag, State.SUCCEEDED); } - } diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery2.java b/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery2.java index 90b6dd068b..67e8525589 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery2.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery2.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,6 +19,7 @@ package org.apache.tez.test; import java.nio.ByteBuffer; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -127,13 +128,13 @@ private TezConfiguration createSessionConfig(Path remoteStagingDir) { } @Before - public void setup() throws Exception { + public void setup() throws Exception { Path remoteStagingDir = remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String .valueOf(new Random().nextInt(100000)))); TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir); TezConfiguration tezConf = createSessionConfig(remoteStagingDir); - + tezSession = TezClient.create("TestDAGRecovery2", tezConf); tezSession.start(); } @@ -172,7 +173,7 @@ void runDAGAndVerify(DAG dag, DAGStatus.State finalState, Assert.assertEquals(finalState, dagStatus.getState()); } - @Test(timeout=120000) + @Test(timeout = 120000) public void testFailingCommitter() throws Exception { DAG dag = SimpleVTestDAG.createDAG("FailingCommitterDAG", null); OutputDescriptor od = @@ -186,7 +187,7 @@ public void testFailingCommitter() throws Exception { runDAGAndVerify(dag, State.FAILED); } - @Test(timeout=120000) + @Test(timeout = 120000) public void testSessionDisableMultiAttempts() throws Exception { tezSession.stop(); Path remoteStagingDir = remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String @@ -203,6 +204,4 @@ public void testSessionDisableMultiAttempts() throws Exception { runDAGAndVerify(dag, State.FAILED, session); session.stop(); } - - } diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestDriver.java b/tez-tests/src/test/java/org/apache/tez/test/TestDriver.java index bf04fd5369..52fa09f080 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestDriver.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestDriver.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,15 +22,14 @@ public class TestDriver { - public static void main(String argv[]){ + public static void main(String argv[]) { int exitCode = -1; ProgramDriver pgd = new ProgramDriver(); try { pgd.addClass("FaultToleranceTestRunner", FaultToleranceTestRunner.class, "Run different DAGs for fault tolerance testing"); exitCode = pgd.run(argv); - } - catch(Throwable e){ + } catch (Throwable e) { e.printStackTrace(); } diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestExceptionPropagation.java b/tez-tests/src/test/java/org/apache/tez/test/TestExceptionPropagation.java index f66bc93f7f..e320040303 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestExceptionPropagation.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestExceptionPropagation.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -213,9 +213,9 @@ private void stopNonSessionClient() { /** * verify the diagnostics in DAGStatus is correct in session mode, using local * mode for fast speed - * + * * @throws Exception - * + * */ @Test(timeout = 600000) public void testExceptionPropagationSession() throws Exception { @@ -243,7 +243,7 @@ public void testExceptionPropagationSession() throws Exception { * verify the diagnostics in {@link DAGStatus} is correct in non-session mode, * and also verify that diagnostics from {@link DAGStatus} should match that * from {@link ApplicationReport} - * + * * @throws Exception */ @Test(timeout = 120000) @@ -311,7 +311,7 @@ public static enum ExceptionLocation { PROCESSOR_COUNTER_EXCEEDED, // VM - VM_INITIALIZE, VM_ON_ROOTVERTEX_INITIALIZE,VM_ON_SOURCETASK_COMPLETED, VM_ON_VERTEX_STARTED, + VM_INITIALIZE, VM_ON_ROOTVERTEX_INITIALIZE, VM_ON_SOURCETASK_COMPLETED, VM_ON_VERTEX_STARTED, VM_ON_VERTEXMANAGEREVENT_RECEIVED, // EdgeManager @@ -323,13 +323,12 @@ public static enum ExceptionLocation { // II II_Initialize, II_HandleInputInitializerEvents, II_OnVertexStateUpdated - } /** * create a DAG with 2 vertices (v1 --> v2), set payload on Input/Output/Processor/VertexManagerPlugin to * control where throw exception - * + * * @param exLocation * @return * @throws IOException @@ -348,18 +347,18 @@ private DAG createDAG(ExceptionLocation exLocation) throws IOException { v1.setVertexManagerPlugin(RootInputVertexManagerWithException .getVMDesc(exLocation)); - Vertex v2 = + Vertex v2 = Vertex.create("v2", DoNothingProcessor.getProcDesc(), 1); v2.addDataSource("input2", DataSourceDescriptor.create(InputDescriptor.create(NoOpInput.class.getName()), - InputInitializerWithException2.getIIDesc(payload), null)); + InputInitializerWithException2.getIIDesc(payload), null)); dag.addVertex(v1) - .addVertex(v2); + .addVertex(v2); if (exLocation.name().startsWith("EM_")) { dag.addEdge(Edge.create(v1, v2, EdgeProperty.create( EdgeManagerPluginDescriptor.create(CustomEdgeManager.class.getName()) - .setUserPayload(payload), + .setUserPayload(payload), DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputWithException.getOutputDesc(payload), InputWithException.getInputDesc(payload)))); } else { @@ -400,7 +399,7 @@ public void handleInputInitializerEvent(List events) public static InputInitializerDescriptor getIIDesc(UserPayload payload) { return InputInitializerDescriptor.create( - InputInitializerWithException.class.getName()) + InputInitializerWithException.class.getName()) .setUserPayload(payload); } } @@ -458,7 +457,7 @@ public void onVertexStateUpdated(VertexStateUpdate stateUpdate) public static InputInitializerDescriptor getIIDesc(UserPayload payload) { return InputInitializerDescriptor.create( - InputInitializerWithException2.class.getName()) + InputInitializerWithException2.class.getName()) .setUserPayload(payload); } } @@ -519,7 +518,7 @@ public List initialize() throws Exception { getContext().requestInitialMemory(0l, null); // mandatory call if (this.exLocation == ExceptionLocation.INPUT_INITIALIZE) { throw new Exception(this.exLocation.name()); - } else if ( getContext().getSourceVertexName().equals("v1")) { + } else if (getContext().getSourceVertexName().equals("v1")) { if (this.exLocation == ExceptionLocation.EM_RouteInputErrorEventToSource || this.exLocation == ExceptionLocation.EM_GetNumDestinationConsumerTasks) { Event errorEvent = InputReadErrorEvent.create("read error", 0, 0); @@ -541,7 +540,7 @@ public static class OutputWithException extends AbstractLogicalOutput { private ExceptionLocation exLocation; public OutputWithException(OutputContext outputContext, - int numPhysicalOutputs) { + int numPhysicalOutputs) { super(outputContext, numPhysicalOutputs); this.exLocation = ExceptionLocation.valueOf(new String(getContext().getUserPayload() @@ -553,7 +552,6 @@ public void start() throws Exception { if (this.exLocation == ExceptionLocation.OUTPUT_START) { throw new Exception(this.exLocation.name()); } - } @Override @@ -620,7 +618,7 @@ public ProcessorWithException(ProcessorContext context) { @Override public void run(Map inputs, - Map outputs) throws Exception { + Map outputs) throws Exception { InputWithException input = (InputWithException) inputs.get("input"); input.start(); input.getReader(); @@ -635,8 +633,8 @@ public void run(Map inputs, throw new Exception(this.exLocation.name()); } else if (this.exLocation == ExceptionLocation.PROCESSOR_COUNTER_EXCEEDED) { // simulate the counter limitation exceeded - for (int i=0;i< TezConfiguration.TEZ_COUNTERS_MAX_DEFAULT+1; ++i) { - getContext().getCounters().findCounter("mycounter", "counter_"+i).increment(1); + for (int i = 0; i < TezConfiguration.TEZ_COUNTERS_MAX_DEFAULT + 1; ++i) { + getContext().getCounters().findCounter("mycounter", "counter_" + i).increment(1); } } } @@ -699,7 +697,7 @@ public void initialize() { @Override public void onRootVertexInitialized(String inputName, - InputDescriptor inputDescriptor, List events) { + InputDescriptor inputDescriptor, List events) { if (this.exLocation == ExceptionLocation.VM_ON_ROOTVERTEX_INITIALIZE) { throw new RuntimeException(this.exLocation.name()); } @@ -728,7 +726,7 @@ public static VertexManagerPluginDescriptor getVMDesc( conf.set(Test_ExceptionLocation, exLocation.name()); UserPayload payload = TezUtils.createUserPayloadFromConf(conf); return VertexManagerPluginDescriptor.create( - RootInputVertexManagerWithException.class.getName()) + RootInputVertexManagerWithException.class.getName()) .setUserPayload(payload); } } @@ -783,7 +781,7 @@ public static VertexManagerPluginDescriptor getVMDesc(ExceptionLocation exLocati conf.set(Test_ExceptionLocation, exLocation.name()); UserPayload payload = TezUtils.createUserPayloadFromConf(conf); return VertexManagerPluginDescriptor.create(InputReadyVertexManagerWithException.class.getName()) - .setUserPayload(payload); + .setUserPayload(payload); } } @@ -841,15 +839,15 @@ public int getNumDestinationTaskPhysicalInputs(int destinationTaskIndex) { @Override public void routeDataMovementEventToDestination(DataMovementEvent event, - int sourceTaskIndex, int sourceOutputIndex, - Map> destinationTaskAndInputIndices) { + int sourceTaskIndex, int sourceOutputIndex, + Map> destinationTaskAndInputIndices) { if (exLocation == ExceptionLocation.EM_RouteDataMovementEventToDestination) { throw new RuntimeException(exLocation.name()); } super.routeDataMovementEventToDestination(event, sourceTaskIndex, sourceOutputIndex, destinationTaskAndInputIndices); } - + @Override public void prepareForRouting() throws Exception { if (exLocation == ExceptionLocation.EM_PrepareForRouting) { @@ -879,7 +877,7 @@ public CompositeEventRouteMetadata routeCompositeDataMovementEventToDestination( @Override public int routeInputErrorEventToSource(InputReadErrorEvent event, - int destinationTaskIndex, int destinationFailedInputIndex) { + int destinationTaskIndex, int destinationFailedInputIndex) { if (exLocation == ExceptionLocation.EM_RouteInputErrorEventToSource) { throw new RuntimeException(exLocation.name()); } @@ -889,7 +887,7 @@ public int routeInputErrorEventToSource(InputReadErrorEvent event, @Override public int routeInputErrorEventToSource(int destinationTaskIndex, - int destinationFailedInputIndex) { + int destinationFailedInputIndex) { if (exLocation == ExceptionLocation.EM_RouteInputErrorEventToSource) { throw new RuntimeException(exLocation.name()); } diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestFaultTolerance.java b/tez-tests/src/test/java/org/apache/tez/test/TestFaultTolerance.java index e2fc53f69a..7eec83754b 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestFaultTolerance.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestFaultTolerance.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -71,7 +71,7 @@ public class TestFaultTolerance { private static String TEST_ROOT_DIR = "target" + Path.SEPARATOR + TestFaultTolerance.class.getName() + "-tmpDir"; protected static MiniDFSCluster dfsCluster; - + private static TezClient tezSession = null; private static TezConfiguration tezConf; @@ -94,11 +94,11 @@ public static void setup() throws Exception { miniTezconf.set("fs.defaultFS", remoteFs.getUri().toString()); // use HDFS miniTezCluster.init(miniTezconf); miniTezCluster.start(); - + Path remoteStagingDir = remoteFs.makeQualified(new Path(TEST_ROOT_DIR, String .valueOf(new Random().nextInt(100000)))); TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir); - + tezConf = new TezConfiguration(miniTezCluster.getConfig()); tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString()); @@ -141,9 +141,9 @@ void runDAGAndVerify(DAG dag, DAGStatus.State finalState) throws Exception { void runDAGAndVerify(DAG dag, DAGStatus.State finalState, int checkFailedAttempts) throws Exception { runDAGAndVerify(dag, finalState, checkFailedAttempts, null); } - - void runDAGAndVerify(DAG dag, DAGStatus.State finalState, int checkFailedAttempts, - String diagnostics) throws Exception { + + void runDAGAndVerify(DAG dag, DAGStatus.State finalState, int checkFailedAttempts, + String diagnostics) throws Exception { tezSession.waitTillReady(); DAGClient dagClient = tezSession.submitDAG(dag); DAGStatus dagStatus = dagClient.getDAGStatus(null); @@ -157,7 +157,7 @@ void runDAGAndVerify(DAG dag, DAGStatus.State finalState, int checkFailedAttempt } Assert.assertEquals(finalState, dagStatus.getState()); - + if (checkFailedAttempts > 0) { Assert.assertEquals(checkFailedAttempts, dagStatus.getDAGProgress().getFailedTaskAttemptCount()); @@ -169,7 +169,7 @@ void runDAGAndVerify(DAG dag, DAGStatus.State finalState, int checkFailedAttempt } } - @Test (timeout=600000) + @Test(timeout = 600000) public void testSessionStopped() throws Exception { Configuration testConf = new Configuration(false); @@ -202,13 +202,13 @@ public void testSessionStopped() throws Exception { tezSession.start(); } - @Test (timeout=60000) + @Test(timeout = 60000) public void testBasicSuccessScatterGather() throws Exception { DAG dag = SimpleTestDAG.createDAG("testBasicSuccessScatterGather", null); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - - @Test (timeout=60000) + + @Test(timeout = 60000) public void testBasicSuccessBroadcast() throws Exception { DAG dag = DAG.create("testBasicSuccessBroadcast"); Vertex v1 = @@ -223,8 +223,8 @@ public void testBasicSuccessBroadcast() throws Exception { TestInput.getInputDesc(null)))); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - - @Test (timeout=60000) + + @Test(timeout = 60000) public void testBasicTaskFailure() throws Exception { Configuration testConf = new Configuration(false); testConf.setBoolean(TestProcessor.getVertexConfName( @@ -233,7 +233,7 @@ public void testBasicTaskFailure() throws Exception { TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "v1"), "0"); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "v1"), 0); - + //verify value at v2 task1 testConf.set(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "1"); @@ -248,8 +248,8 @@ public void testBasicTaskFailure() throws Exception { DAG dag = SimpleTestDAG.createDAG("testBasicTaskFailure", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED, 1); } - - @Test (timeout=60000) + + @Test(timeout = 60000) public void testTaskMultipleFailures() throws Exception { Configuration testConf = new Configuration(false); testConf.setBoolean(TestProcessor.getVertexConfName( @@ -258,19 +258,19 @@ public void testTaskMultipleFailures() throws Exception { TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "v1"), "0,1"); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "v1"), 1); - + //v1 task0,1 attempt 2 succeed. Input sum = 6. Plus one (v2 attempt0). //ending sum is 7. testConf.set(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0"); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0"); testConf.setInt(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 0), 7); - + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 0), 7); + DAG dag = SimpleTestDAG.createDAG("testTaskMultipleFailures", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED, 4); } - - @Test (timeout=60000) + + @Test(timeout = 60000) public void testTaskMultipleFailuresDAGFail() throws Exception { Configuration testConf = new Configuration(false); testConf.setBoolean(TestProcessor.getVertexConfName( @@ -279,12 +279,12 @@ public void testTaskMultipleFailuresDAGFail() throws Exception { TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "v1"), "0"); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "v1"), -1); - + DAG dag = SimpleTestDAG.createDAG("testTaskMultipleFailuresDAGFail", testConf); runDAGAndVerify(dag, DAGStatus.State.FAILED); } - - @Test (timeout=60000) + + @Test(timeout = 60000) public void testBasicInputFailureWithExit() throws Exception { Configuration testConf = new Configuration(false); testConf.setBoolean(TestInput.getVertexConfName( @@ -297,22 +297,22 @@ public void testBasicInputFailureWithExit() throws Exception { TestInput.TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, "v2"), "0"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v2"), "0"); - + //v2 task1 attempt0 index0 fails and exits. //v1 task0 attempt1 reruns. //v2 task1 attempt1 has: // v1 task0 attempt1 (value = 2) + v1 task1 attempt0 (value = 1) // + its own value, attempt + 1 (value = 2). Total is 5. testConf.set(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0,1"); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0,1"); testConf.setInt(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 5); - + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 5); + DAG dag = SimpleTestDAG.createDAG("testBasicInputFailureWithExit", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - - @Test (timeout=60000) + + @Test(timeout = 60000) public void testBasicInputFailureWithoutExit() throws Exception { Configuration testConf = new Configuration(false); testConf.setBoolean(TestInput.getVertexConfName( @@ -323,18 +323,18 @@ public void testBasicInputFailureWithoutExit() throws Exception { TestInput.TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, "v2"), "0"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v2"), "0"); - + testConf.set(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0,1"); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0,1"); testConf.setInt(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 4); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 4); testConf.setInt(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 0), 3); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 0), 3); DAG dag = SimpleTestDAG.createDAG("testBasicInputFailureWithoutExit", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - - @Test (timeout=60000) + + @Test(timeout = 60000) public void testBasicInputFailureWithoutExitDeadline() throws Exception { Configuration testConf = new Configuration(false); testConf.setInt(SimpleTestDAG.TEZ_SIMPLE_DAG_NUM_TASKS, 3); // 1 error < 0.4 fail fraction @@ -346,13 +346,12 @@ public void testBasicInputFailureWithoutExitDeadline() throws Exception { TestInput.TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, "v2"), "0"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v2"), "0"); - + DAG dag = SimpleTestDAG.createDAG("testBasicInputFailureWithoutExitDeadline", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - - @Test (timeout=60000) + @Test(timeout = 60000) public void testMultipleInputFailureWithoutExit() throws Exception { Configuration testConf = new Configuration(false); testConf.setBoolean(TestInput.getVertexConfName( @@ -363,23 +362,23 @@ public void testMultipleInputFailureWithoutExit() throws Exception { TestInput.TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, "v2"), "0"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v2"), "-1"); - + //v2 task0 attempt0 input0,1 fails. wait. //v1 task0 attempt1 reruns. v1 task1 attempt1 reruns. //2 + 2 + 1 = 5 //same number for v2 task1 testConf.set(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0,1"); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0,1"); testConf.setInt(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 0), 5); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 0), 5); testConf.setInt(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 5); - + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 5); + DAG dag = SimpleTestDAG.createDAG("testMultipleInputFailureWithoutExit", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - - @Test (timeout=60000) + + @Test(timeout = 60000) public void testMultiVersionInputFailureWithoutExit() throws Exception { Configuration testConf = new Configuration(false); testConf.setBoolean(TestInput.getVertexConfName( @@ -392,66 +391,66 @@ public void testMultiVersionInputFailureWithoutExit() throws Exception { TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v2"), "0"); testConf.setInt(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v2"), 1); - + //v2 task1 attempt0 input0 input-attempt0 fails. Wait. v1 task0 attempt1 reruns. //v2 task1 attempt0 input0 input-attempt1 fails. Wait. v1 task0 attempt2 reruns. //v2 task1 attempt0 input0 input-attempt2 succeeds. //input values (3 + 1) + 1 = 5 testConf.set(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0,1"); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0,1"); testConf.setInt(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 5); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 5); testConf.setInt(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 0), 3); - + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 0), 3); + DAG dag = SimpleTestDAG.createDAG("testMultiVersionInputFailureWithoutExit", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - - @Test (timeout=60000) + + @Test(timeout = 60000) public void testTwoLevelsFailingDAGSuccess() throws Exception { Configuration testConf = new Configuration(); DAG dag = TwoLevelsFailingDAG.createDAG("testTwoLevelsFailingDAGSuccess", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - - @Test (timeout=60000) + + @Test(timeout = 60000) public void testThreeLevelsFailingDAGSuccess() throws Exception { Configuration testConf = new Configuration(); DAG dag = ThreeLevelsFailingDAG.createDAG("testThreeLevelsFailingDAGSuccess", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - - @Test (timeout=60000) + + @Test(timeout = 60000) public void testSixLevelsFailingDAGSuccess() throws Exception { Configuration testConf = new Configuration(); DAG dag = SixLevelsFailingDAG.createDAG("testSixLevelsFailingDAGSuccess", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - - @Test (timeout=60000) + + @Test(timeout = 60000) public void testThreeLevelsFailingDAG2VerticesHaveFailedAttemptsDAGSucceeds() throws Exception { Configuration testConf = new Configuration(); //set maximum number of task attempts to 4 testConf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 4); //l2v1 failure testConf.setBoolean(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_DO_FAIL, "l2v1"), true); + TestProcessor.TEZ_FAILING_PROCESSOR_DO_FAIL, "l2v1"), true); testConf.set(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "l2v1"), "1"); + TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "l2v1"), "1"); //3 attempts fail testConf.setInt(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "l2v1"), 2); - + TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "l2v1"), 2); + //l3v1 failure testConf.setBoolean(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_DO_FAIL, "l3v1"), true); + TestProcessor.TEZ_FAILING_PROCESSOR_DO_FAIL, "l3v1"), true); testConf.set(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "l3v1"), "0"); + TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "l3v1"), "0"); //3 attempts fail testConf.setInt(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "l3v1"), 2); - + TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "l3v1"), 2); + //l2v1: task0 attempt0 succeeds. task1 attempt3 succeeds. //l3v1 finally task0 attempt3 will succeed. //l1v1 outputs 1. l1v2 outputs 2. @@ -459,20 +458,20 @@ public void testThreeLevelsFailingDAG2VerticesHaveFailedAttemptsDAGSucceeds() th //l2v2 output: attempt0 (l1v2+self = 2+1) * 3 tasks = 9 //l3v1 task0 attempt3 = l2v1 (2) + l2v2 (9) + self (4) = 15 testConf.set(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "l3v1"), "0"); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "l3v1"), "0"); testConf.setInt(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "l3v1", 0), 15); - + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "l3v1", 0), 15); + DAG dag = ThreeLevelsFailingDAG.createDAG("testThreeLevelsFailingDAG2VerticesHaveFailedAttemptsDAGSucceeds", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + /** * Test input failure. * v1-task0 v1-task1 * | \ / | * v2-task0 v2-task1 - * + * * Use maximum allowed failed attempt of 4 (default value during session creation). * v1-task1-attempt0 fails. Attempt 1 succeeds. * v2-task0-attempt0 runs. Its input1-inputversion0 fails. @@ -482,12 +481,12 @@ public void testThreeLevelsFailingDAG2VerticesHaveFailedAttemptsDAGSucceeds() th * The input version is now 2. The attempt will now succeed. * @throws Exception */ - @Test (timeout=60000) + @Test(timeout = 60000) public void testInputFailureCausesRerunAttemptWithinMaxAttemptSuccess() throws Exception { Configuration testConf = new Configuration(); //at v1, task 1 has attempt 0 failing. Attempt 1 succeeds. 1 attempt fails so far. testConf.setBoolean(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_DO_FAIL, "v1"), true); + TestProcessor.TEZ_FAILING_PROCESSOR_DO_FAIL, "v1"), true); testConf.set(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "v1"), "1"); testConf.setInt(TestProcessor.getVertexConfName( @@ -508,65 +507,65 @@ public void testInputFailureCausesRerunAttemptWithinMaxAttemptSuccess() throws E //at v2-task0-attempt0/1-input1 has input failure at input version 0 only. testConf.setInt(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v2"), 1); - + //v2-task1-attempt0 takes v1-task0-attempt0 input and v1-task1-attempt1 input. //v2-task1 does not take v1-task1-attempt2 (re-run caused by input failure //triggered by v2-task0) output. //1 + 2 + 1 = 4 testConf.set(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0,1"); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0,1"); testConf.setInt(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 0), 5); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 0), 5); // Work-around till TEZ-877 gets fixed //testConf.setInt(TestProcessor.getVertexConfName( // TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 4); - + DAG dag = SimpleTestDAG.createDAG( - "testInputFailureCausesRerunAttemptWithinMaxAttemptSuccess", testConf); + "testInputFailureCausesRerunAttemptWithinMaxAttemptSuccess", testConf); //Job should succeed. runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + /** * Sets configuration for cascading input failure tests that * use SimpleTestDAG3Vertices. * @param testConf configuration * @param failAndExit whether input failure should trigger attempt exit */ - private void setCascadingInputFailureConfig(Configuration testConf, + private void setCascadingInputFailureConfig(Configuration testConf, boolean failAndExit) { // v2 attempt0 succeeds. // v2 task0 attempt1 input0 fails up to version 0. testConf.setBoolean(TestInput.getVertexConfName( - TestInput.TEZ_FAILING_INPUT_DO_FAIL, "v2"), true); + TestInput.TEZ_FAILING_INPUT_DO_FAIL, "v2"), true); testConf.setBoolean(TestInput.getVertexConfName( - TestInput.TEZ_FAILING_INPUT_DO_FAIL_AND_EXIT, "v2"), failAndExit); + TestInput.TEZ_FAILING_INPUT_DO_FAIL_AND_EXIT, "v2"), failAndExit); testConf.set(TestInput.getVertexConfName( - TestInput.TEZ_FAILING_INPUT_FAILING_TASK_INDEX, "v2"), "0"); + TestInput.TEZ_FAILING_INPUT_FAILING_TASK_INDEX, "v2"), "0"); testConf.set(TestInput.getVertexConfName( - TestInput.TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, "v2"), "1"); + TestInput.TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, "v2"), "1"); testConf.set(TestInput.getVertexConfName( - TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v2"), "0"); + TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v2"), "0"); testConf.setInt(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v2"), - 0); + 0); //v3 all-tasks attempt0 input0 fails up to version 0. testConf.setBoolean(TestInput.getVertexConfName( - TestInput.TEZ_FAILING_INPUT_DO_FAIL, "v3"), true); + TestInput.TEZ_FAILING_INPUT_DO_FAIL, "v3"), true); testConf.setBoolean(TestInput.getVertexConfName( - TestInput.TEZ_FAILING_INPUT_DO_FAIL_AND_EXIT, "v3"), failAndExit); + TestInput.TEZ_FAILING_INPUT_DO_FAIL_AND_EXIT, "v3"), failAndExit); testConf.set(TestInput.getVertexConfName( - TestInput.TEZ_FAILING_INPUT_FAILING_TASK_INDEX, "v3"), "-1"); + TestInput.TEZ_FAILING_INPUT_FAILING_TASK_INDEX, "v3"), "-1"); testConf.set(TestInput.getVertexConfName( - TestInput.TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, "v3"), "0"); + TestInput.TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, "v3"), "0"); testConf.set(TestInput.getVertexConfName( - TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v3"), "0"); + TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v3"), "0"); testConf.setInt(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v3"), - 0); + 0); } - + /** * Test cascading input failure without exit. Expecting success. * v1 -- v2 -- v3 @@ -574,29 +573,29 @@ private void setCascadingInputFailureConfig(Configuration testConf, * v2 task0 attempt1 input0 fails. Wait. Triggering v1 rerun. * v1 attempt1 rerun and succeeds. v2 accepts v1 attempt1 output. v2 attempt1 succeeds. * v3 attempt0 accepts v2 attempt1 output. - * + * * AM vertex succeeded order is v1, v2, v1, v2, v3. * @throws Exception */ - @Test (timeout=60000) + @Test(timeout = 60000) public void testCascadingInputFailureWithoutExitSuccess() throws Exception { Configuration testConf = new Configuration(false); setCascadingInputFailureConfig(testConf, false); - + //v2 task0 attempt1 value = v1 task0 attempt1 (2) + v1 task1 attempt0 (1) + 2 = 5 //v3 all-tasks attempt0 takes v2 task0 attempt1 value (5) + v2 task1 attempt0 (3) + 1 = 9 testConf.set(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v3"), "0,1"); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v3"), "0,1"); testConf.setInt(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 9); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 9); testConf.setInt(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 1), 9); - + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 1), 9); + DAG dag = SimpleTestDAG3Vertices.createDAG( - "testCascadingInputFailureWithoutExitSuccess", testConf); + "testCascadingInputFailureWithoutExitSuccess", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + /** * Test cascading input failure with exit. Expecting success. * v1 -- v2 -- v3 @@ -604,38 +603,38 @@ public void testCascadingInputFailureWithoutExitSuccess() throws Exception { * v2 task0 attempt1 input0 fails. v2 attempt1 exits. Triggering v1 rerun. * v1 attempt1 rerun and succeeds. v2 accepts v1 attempt1 output. v2 attempt2 succeeds. * v3 attempt1 accepts v2 attempt2 output. - * + * * AM vertex succeeded order is v1, v2, v3, v1, v2, v3. * @throws Exception */ - @Test (timeout=60000) + @Test(timeout = 60000) public void testCascadingInputFailureWithExitSuccess() throws Exception { Configuration testConf = new Configuration(false); setCascadingInputFailureConfig(testConf, true); - + //v2 task0 attempt2 value = v1 task0 attempt1 (2) + v1 task1 attempt0 (1) + 3 = 6 //v3 all-tasks attempt1 takes v2 task0 attempt2 value (6) + v2 task1 attempt0 (3) + 2 = 11 testConf.set(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v3"), "0,1"); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v3"), "0,1"); testConf.setInt(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 11); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 11); testConf.setInt(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 1), 11); - + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 1), 11); + DAG dag = SimpleTestDAG3Vertices.createDAG( - "testCascadingInputFailureWithExitSuccess", testConf); + "testCascadingInputFailureWithExitSuccess", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + /** * Input failure of v3 causes rerun of both both v1 and v2 vertices. * v1 v2 * \ / * v3 - * + * * @throws Exception */ - @Test (timeout=60000) + @Test(timeout = 60000) public void testInputFailureCausesRerunOfTwoVerticesWithoutExit() throws Exception { Configuration testConf = new Configuration(false); testConf.setBoolean(TestInput.getVertexConfName( @@ -650,65 +649,65 @@ public void testInputFailureCausesRerunOfTwoVerticesWithoutExit() throws Excepti TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v3"), "-1"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v3"), "1"); - + //v3 attempt0: //v1 task0,1 attempt2 = 6. v2 task0,1 attempt2 = 6. //total = 6 + 6 + 1 = 13 testConf.set(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v3"), "0"); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v3"), "0"); testConf.setInt(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 13); - + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 13); + DAG dag = SimpleVTestDAG.createDAG( - "testInputFailureCausesRerunOfTwoVerticesWithoutExit", testConf); + "testInputFailureCausesRerunOfTwoVerticesWithoutExit", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + /** * Downstream(v3) attempt failure of a vertex connected with * 2 upstream vertices.. * v1 v2 * \ / * v3 - * + * * @throws Exception */ - @Test (timeout=60000) + @Test(timeout = 60000) public void testAttemptOfDownstreamVertexConnectedWithTwoUpstreamVerticesFailure() throws Exception { Configuration testConf = new Configuration(false); - + testConf.setBoolean(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_DO_FAIL, "v3"), true); testConf.set(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_TASK_INDEX, "v3"), "0,1"); testConf.setInt(TestProcessor.getVertexConfName( TestProcessor.TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, "v3"), 1); - + //v1 input = 2. v2 input = 2 //v3 attempt2 value = 2 + 2 + 3 = 7 testConf.set(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v3"), "0,1"); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v3"), "0,1"); testConf.setInt(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 7); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 7); testConf.setInt(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 1), 7); - + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 1), 7); + DAG dag = SimpleVTestDAG.createDAG( - "testAttemptOfDownstreamVertexConnectedWithTwoUpstreamVerticesFailure", testConf); + "testAttemptOfDownstreamVertexConnectedWithTwoUpstreamVerticesFailure", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + /** * Input failure of v2,v3 trigger v1 rerun. * Reruns can send output to 2 downstream vertices. * v1 * / \ * v2 v3 - * + * * Also covers multiple consumer vertices report failure against same producer task. * @throws Exception */ - @Test (timeout=60000) + @Test(timeout = 60000) public void testInputFailureRerunCanSendOutputToTwoDownstreamVertices() throws Exception { Configuration testConf = new Configuration(false); testConf.setBoolean(TestInput.getVertexConfName( @@ -723,7 +722,7 @@ public void testInputFailureRerunCanSendOutputToTwoDownstreamVertices() throws E TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v2"), "-1"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v2"), "0"); - + testConf.setBoolean(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_DO_FAIL, "v3"), true); testConf.setBoolean(TestInput.getVertexConfName( @@ -736,36 +735,35 @@ public void testInputFailureRerunCanSendOutputToTwoDownstreamVertices() throws E TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v3"), "-1"); testConf.set(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v3"), "0"); - + //both vertices trigger v1 rerun. v1 attempt1 output is 2 * 2 tasks = 4. //v2 attempt0 = 4 + 1 = 5 testConf.set(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0"); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0"); testConf.setInt(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 0), 5); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 0), 5); //v3 attempt0 = 4 + 1 = 5 testConf.set(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v3"), "0"); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v3"), "0"); testConf.setInt(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 5); - + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3", 0), 5); + DAG dag = SimpleReverseVTestDAG.createDAG( - "testInputFailureRerunCanSendOutputToTwoDownstreamVertices", testConf); + "testInputFailureRerunCanSendOutputToTwoDownstreamVertices", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - - + /** * SimpleTestDAG (v1,v2) has v2 task0/1 input failures triggering v1 rerun * upto version 1. - * + * * v1 attempt0 succeeds. * v2-task0-attempt0 rejects v1 version0/1. Trigger v1 attempt1. * v2-task1-attempt0 rejects v1 version0/1. Trigger v1 attempt2. * DAG succeeds with v1 attempt2. * @throws Exception */ - @Test (timeout=60000) + @Test(timeout = 60000) public void testTwoTasksHaveInputFailuresSuccess() throws Exception { Configuration testConf = new Configuration(false); testConf.setBoolean(TestInput.getVertexConfName( @@ -780,23 +778,23 @@ public void testTwoTasksHaveInputFailuresSuccess() throws Exception { TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v2"), "0"); testConf.setInt(TestInput.getVertexConfName( TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v2"), 1); - + //v2 task0 accepts v1 task0 attempt2(3) and v1 task1 attempt0(1) = 4 //v2 task0 attempt0 = 1 //total = 5 testConf.set(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0,1"); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v2"), "0,1"); testConf.setInt(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 0), 5); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 0), 5); //similarly for v2 task1 testConf.setInt(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 5); - + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v2", 1), 5); + DAG dag = SimpleTestDAG.createDAG("testTwoTasksHaveInputFailuresSuccess", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - - @Test (timeout=240000) + + @Test(timeout = 240000) public void testRandomFailingTasks() throws Exception { Configuration testConf = new Configuration(false); testConf.setBoolean(TestProcessor.TEZ_FAILING_PROCESSOR_DO_RANDOM_FAIL, true); @@ -804,9 +802,9 @@ public void testRandomFailingTasks() throws Exception { DAG dag = SixLevelsFailingDAG.createDAG("testRandomFailingTasks", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - + @Ignore - @Test (timeout=240000) + @Test(timeout = 240000) public void testRandomFailingInputs() throws Exception { Configuration testConf = new Configuration(false); testConf.setBoolean(TestInput.TEZ_FAILING_INPUT_DO_RANDOM_FAIL, true); @@ -814,18 +812,18 @@ public void testRandomFailingInputs() throws Exception { DAG dag = SixLevelsFailingDAG.createDAG("testRandomFailingInputs", testConf); runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED); } - - @Test (timeout=240000) + + @Test(timeout = 240000) public void testNoProgress() throws Exception { Configuration testConf = new Configuration(false); testConf.setInt(SimpleTestDAG.TEZ_SIMPLE_DAG_NUM_TASKS, 1); testConf.setLong(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_SLEEP_MS, "v1"), 1000*100); // long sleep + TestProcessor.TEZ_FAILING_PROCESSOR_SLEEP_MS, "v1"), 1000 * 100); // long sleep DAG dag = SimpleTestDAG.createDAG(testConf); Vertex hung = dag.getVertex("v1"); hung.setConf(TezConfiguration.TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS, Long.toString(1000)); hung.setConf(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, Integer.toString(2)); - + // dag will fail with 2 attempts failing from vertex v1 runDAGAndVerify(dag, DAGStatus.State.FAILED, 2, "no progress"); } @@ -845,49 +843,49 @@ public void testCartesianProduct() throws Exception { Configuration vertexConf = new Configuration(); vertexConf.setInt(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v3"), 3); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX, "v3"), 3); vertexConf.setInt(TestProcessor.getVertexConfName( - TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3"), 5); + TestProcessor.TEZ_FAILING_PROCESSOR_VERIFY_VALUE, "v3"), 5); UserPayload vertexPayload = TezUtils.createUserPayloadFromConf(vertexConf); ProcessorDescriptor processorDescriptor = - ProcessorDescriptor.create(TestProcessor.class.getName()).setUserPayload(vertexPayload); + ProcessorDescriptor.create(TestProcessor.class.getName()).setUserPayload(vertexPayload); Vertex v1 = Vertex.create("v1", processorDescriptor, 2); Vertex v2 = Vertex.create("v2", processorDescriptor, 2); Vertex v3 = Vertex.create("v3", processorDescriptor); String[] sourceVertices = {"v1", "v2"}; CartesianProductConfig cartesianProductConfig = - new CartesianProductConfig(Arrays.asList(sourceVertices)); + new CartesianProductConfig(Arrays.asList(sourceVertices)); TezConfiguration tezConf = new TezConfiguration(); tezConf.setInt(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_NUM_PARTITIONS, 1); tezConf.setBoolean(CartesianProductVertexManager.TEZ_CARTESIAN_PRODUCT_ENABLE_GROUPING, false); UserPayload cartesianProductPayload = - cartesianProductConfig.toUserPayload(tezConf); + cartesianProductConfig.toUserPayload(tezConf); v3.setVertexManagerPlugin( - VertexManagerPluginDescriptor.create(CartesianProductVertexManager.class.getName()) - .setUserPayload(cartesianProductPayload)); + VertexManagerPluginDescriptor.create(CartesianProductVertexManager.class.getName()) + .setUserPayload(cartesianProductPayload)); EdgeManagerPluginDescriptor edgeManagerPluginDescriptor = - EdgeManagerPluginDescriptor.create(CartesianProductEdgeManager.class.getName()) - .setUserPayload(cartesianProductPayload); + EdgeManagerPluginDescriptor.create(CartesianProductEdgeManager.class.getName()) + .setUserPayload(cartesianProductPayload); Configuration inputConf = new Configuration(); inputConf.setBoolean(TestInput.getVertexConfName( - TestInput.TEZ_FAILING_INPUT_DO_FAIL, "v3"), true); + TestInput.TEZ_FAILING_INPUT_DO_FAIL, "v3"), true); inputConf.setInt(TestInput.getVertexConfName( - TestInput.TEZ_FAILING_INPUT_FAILING_TASK_INDEX, "v3"), 3); + TestInput.TEZ_FAILING_INPUT_FAILING_TASK_INDEX, "v3"), 3); inputConf.setInt(TestInput.getVertexConfName( - TestInput.TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, "v3"), 0); + TestInput.TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, "v3"), 0); inputConf.setInt(TestInput.getVertexConfName( - TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v3"), 0); + TestInput.TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, "v3"), 0); inputConf.setInt(TestInput.getVertexConfName( - TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v3"), 0); + TestInput.TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, "v3"), 0); UserPayload inputPayload = TezUtils.createUserPayloadFromConf(inputConf); EdgeProperty edgeProperty = - EdgeProperty.create(edgeManagerPluginDescriptor, DataMovementType.CUSTOM, - DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, TestOutput.getOutputDesc(null), - TestInput.getInputDesc(inputPayload)); + EdgeProperty.create(edgeManagerPluginDescriptor, DataMovementType.CUSTOM, + DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, TestOutput.getOutputDesc(null), + TestInput.getInputDesc(inputPayload)); Edge e1 = Edge.create(v1, v3, edgeProperty); Edge e2 = Edge.create(v2, v3, edgeProperty); dag.addVertex(v1).addVertex(v2).addVertex(v3); diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestInput.java b/tez-tests/src/test/java/org/apache/tez/test/TestInput.java index 36bb983d1d..b0dabfa671 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestInput.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestInput.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.test; @@ -57,7 +57,7 @@ public class TestInput extends AbstractLogicalInput { private static final Logger LOG = LoggerFactory .getLogger(TestInput.class); - + public static final String COUNTER_NAME = "TestInput"; Configuration conf; @@ -66,7 +66,7 @@ public class TestInput extends AbstractLogicalInput { AtomicInteger inputReady = new AtomicInteger(-1); int lastInputReadyValue = -1; int failingInputUpto = 0; - + boolean doFail = false; boolean doRandomFail = false; float randomFailProbability = 0.0f; @@ -76,7 +76,7 @@ public class TestInput extends AbstractLogicalInput { Set failingInputIndices = Sets.newHashSet(); Integer failAll = new Integer(-1); int[] inputValues; - + /** * Enable failure for this logical input */ @@ -130,7 +130,7 @@ public TestInput(InputContext inputContext, int numPhysicalInputs) { super(inputContext, numPhysicalInputs); this.completedInputVersion = new int[numPhysicalInputs]; this.inputValues = new int[numPhysicalInputs]; - for (int i=0; i events = Lists.newLinkedList(); if (failingInputIndices.contains(failAll)) { - for (int i=0; i= getNumPhysicalInputs()) { - throwException("InputIndex: " + index.intValue() + + throwException("InputIndex: " + index.intValue() + " should be less than numInputs: " + getNumPhysicalInputs()); } if (completedInputVersion[index.intValue()] < lastInputReadyValue) { continue; // dont fail a previous version now. } - String msg = ("FailingInput: " + getContext().getUniqueIdentifier() + + String msg = ("FailingInput: " + getContext().getUniqueIdentifier() + " index: " + index.intValue() + " version: " + lastInputReadyValue); events.add(InputReadErrorEvent.create(msg, index.intValue(), lastInputReadyValue, 1, false, false, "localhost")); @@ -220,19 +220,19 @@ public int doRead() { done = false; } } else if ((failingTaskIndices.contains(failAll) || - failingTaskIndices.contains(getContext().getTaskIndex()))){ + failingTaskIndices.contains(getContext().getTaskIndex()))) { boolean previousAttemptReadFailed = false; if (failingTaskAttempts.contains(failAll)) { previousAttemptReadFailed = true; } else { - for (int i=0 ; i events = Lists.newLinkedList(); - for (int index=0; index initialize() throws Exception { getVertexConfName(TEZ_FAILING_INPUT_DO_FAIL_AND_EXIT, vName), false); LOG.info("doFail: " + doFail + " doFailAndExit: " + doFailAndExit); if (doFail) { - for (String failingIndex : - conf.getTrimmedStringCollection( - getVertexConfName(TEZ_FAILING_INPUT_FAILING_TASK_INDEX, vName))) { + for (String failingIndex : + conf.getTrimmedStringCollection( + getVertexConfName(TEZ_FAILING_INPUT_FAILING_TASK_INDEX, vName))) { LOG.info("Adding failing task index: " + failingIndex); failingTaskIndices.add(Integer.valueOf(failingIndex)); } - for (String failingIndex : - conf.getTrimmedStringCollection( - getVertexConfName(TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, vName))) { + for (String failingIndex : + conf.getTrimmedStringCollection( + getVertexConfName(TEZ_FAILING_INPUT_FAILING_TASK_ATTEMPT, vName))) { LOG.info("Adding failing task attempt: " + failingIndex); failingTaskAttempts.add(Integer.valueOf(failingIndex)); } failingInputUpto = conf.getInt( getVertexConfName(TEZ_FAILING_INPUT_FAILING_UPTO_INPUT_ATTEMPT, vName), 0); LOG.info("Adding failing input upto: " + failingInputUpto); - for (String failingIndex : - conf.getTrimmedStringCollection( - getVertexConfName(TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, vName))) { + for (String failingIndex : + conf.getTrimmedStringCollection( + getVertexConfName(TEZ_FAILING_INPUT_FAILING_INPUT_INDEX, vName))) { LOG.info("Adding failing input index: " + failingIndex); failingInputIndices.add(Integer.valueOf(failingIndex)); } @@ -359,7 +358,7 @@ public void handleEvents(List inputEvents) throws Exception { " numInputs: " + getNumPhysicalInputs() + " numCompletedInputs: " + numCompletedInputs); this.completedInputVersion[dmEvent.getTargetIndex()] = dmEvent.getVersion(); - this.inputValues[dmEvent.getTargetIndex()] = + this.inputValues[dmEvent.getTargetIndex()] = dmEvent.getUserPayload().getInt(); } else if (event instanceof InputFailedEvent) { InputFailedEvent ifEvent = (InputFailedEvent) event; @@ -373,8 +372,8 @@ public void handleEvents(List inputEvents) throws Exception { } } if (numCompletedInputs == getNumPhysicalInputs()) { - int maxInputVersionSeen = -1; - for (int i=0; i inputEvents) throws Exception { @Override public List close() throws Exception { - getContext().getCounters().findCounter(COUNTER_NAME, COUNTER_NAME).increment(1);; + getContext().getCounters().findCounter(COUNTER_NAME, COUNTER_NAME).increment(1); + ; return null; } - } diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestLocalMode.java b/tez-tests/src/test/java/org/apache/tez/test/TestLocalMode.java index bdb71ad177..e22faf9fea 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestLocalMode.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestLocalMode.java @@ -163,7 +163,6 @@ public void testMultipleClientsWithoutSession() throws TezException, Interrupted dagClient1.close(); tezClient1.stop(); - TezConfiguration tezConf2 = createConf(); DAG dag2 = createSimpleDAG("dag2", SleepProcessor.class.getName()); TezClient tezClient2 = TezClient.create("commonName", tezConf2, false); @@ -240,7 +239,6 @@ public void close() throws Exception { public void run(Map inputs, Map outputs) throws Exception { throw new TezException("FailingProcessor"); - } } @@ -249,13 +247,13 @@ private DAG createSimpleDAG(String dagName, String processorName) { processorName).setUserPayload( new SleepProcessor.SleepProcessorConfig(1).toUserPayload()), 1)); return dag; - } - @Test(timeout=30000) + + @Test(timeout = 30000) public void testMultiDAGsOnSession() throws IOException, TezException, InterruptedException { int dags = 2;//two dags will be submitted to session String[] inputPaths = new String[dags]; - String[] outputPaths = new String[dags]; + String[] outputPaths = new String[dags]; DAGClient[] dagClients = new DAGClient[dags]; TezConfiguration tezConf = createConf(); @@ -264,7 +262,7 @@ public void testMultiDAGsOnSession() throws IOException, TezException, Interrupt //create inputs and outputs FileSystem fs = FileSystem.get(tezConf); - for(int i = 0; i < dags; i++) { + for (int i = 0; i < dags; i++) { inputPaths[i] = new Path(STAGING_DIR.getAbsolutePath(), "in-" + i).toString(); createInputFile(fs, inputPaths[i]); outputPaths[i] = new Path(STAGING_DIR.getAbsolutePath(), "out-" + i).toString(); @@ -272,7 +270,7 @@ public void testMultiDAGsOnSession() throws IOException, TezException, Interrupt //start testing try { - for (int i=0; i0) { - assertTrue(dagClients[i-1].getExecutionContext().equals(dagClients[i].getExecutionContext())); + if (i > 0) { + assertTrue(dagClients[i - 1].getExecutionContext().equals(dagClients[i].getExecutionContext())); } } } finally { diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestMiniTezCluster.java b/tez-tests/src/test/java/org/apache/tez/test/TestMiniTezCluster.java index 4fda977a4c..d95cbe59f8 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestMiniTezCluster.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestMiniTezCluster.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.test; import java.io.IOException; diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestOutput.java b/tez-tests/src/test/java/org/apache/tez/test/TestOutput.java index 8b292ab196..bfe1ae069f 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestOutput.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestOutput.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.test; @@ -40,6 +40,7 @@ public class TestOutput extends AbstractLogicalOutput { private static final Logger LOG = LoggerFactory.getLogger(TestOutput.class); public static final String COUNTER_NAME = "TestOutput"; + public TestOutput(OutputContext outputContext, int numPhysicalOutputs) { super(outputContext, numPhysicalOutputs); } @@ -51,15 +52,15 @@ public static OutputDescriptor getOutputDesc(UserPayload payload) { } return desc; } - + int output; - + @Override public List initialize() throws Exception { getContext().requestInitialMemory(0l, null); //Mandatory call return Collections.emptyList(); } - + void write(int value) { this.output = value; } @@ -80,7 +81,8 @@ public void handleEvents(List outputEvents) { @Override public List close() throws Exception { LOG.info("Sending data movement event with value: " + output); - getContext().getCounters().findCounter(COUNTER_NAME, COUNTER_NAME).increment(1);; + getContext().getCounters().findCounter(COUNTER_NAME, COUNTER_NAME).increment(1); + ; ByteBuffer result = ByteBuffer.allocate(4).putInt(output); result.flip(); List events = Lists.newArrayListWithCapacity(getNumPhysicalOutputs()); @@ -90,14 +92,13 @@ public List close() throws Exception { } ShuffleUserPayloads.VertexManagerEventPayloadProto.Builder vmBuilder = - ShuffleUserPayloads.VertexManagerEventPayloadProto.newBuilder().setNumRecord(1); + ShuffleUserPayloads.VertexManagerEventPayloadProto.newBuilder().setNumRecord(1); VertexManagerEvent vmEvent = VertexManagerEvent.create( - getContext().getDestinationVertexName(), - vmBuilder.build().toByteString().asReadOnlyByteBuffer()); + getContext().getDestinationVertexName(), + vmBuilder.build().toByteString().asReadOnlyByteBuffer()); events.add(vmEvent); return events; } - } diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestPipelinedShuffle.java b/tez-tests/src/test/java/org/apache/tez/test/TestPipelinedShuffle.java index e44b6eb7f5..4852f31deb 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestPipelinedShuffle.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestPipelinedShuffle.java @@ -142,16 +142,15 @@ private void test(Configuration conf) throws Exception { PipelinedShuffleJob pipelinedShuffle = new PipelinedShuffleJob(); pipelinedShuffle.setConf(conf); - String[] args = new String[] { }; + String[] args = new String[]{}; assertEquals(0, pipelinedShuffle.run(args)); } /** - * * mapper1 --\ - * --> reducer + * --> reducer * mapper2 --/ - * + *

    * Mappers just generate dummy data, but ensures that there is enough spills. * Reducer should process them correctly and validate the total number of records. * Only record count is validated in the reducer which is fine for this test. @@ -165,7 +164,8 @@ public DataGenerator(ProcessorContext context) { super(context); } - @Override public void run() throws Exception { + @Override + public void run() throws Exception { Preconditions.checkArgument(getInputs().size() == 0); Preconditions.checkArgument(getOutputs().size() == 1); KeyValueWriter writer = (KeyValueWriter) getOutputs().get("reducer").getWriter(); @@ -267,5 +267,4 @@ public int run(String[] args) throws Exception { return 0; } } - } diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestProcessor.java b/tez-tests/src/test/java/org/apache/tez/test/TestProcessor.java index b53dad97c7..dbf171ffbc 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestProcessor.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestProcessor.java @@ -1,20 +1,20 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.tez.test; @@ -49,9 +49,9 @@ public class TestProcessor extends AbstractLogicalIOProcessor { private static final Logger LOG = LoggerFactory .getLogger(TestProcessor.class); - + Configuration conf; - + boolean doFail = false; boolean doRandomFail = false; float randomFailProbability = 0.0f; @@ -59,10 +59,10 @@ public class TestProcessor extends AbstractLogicalIOProcessor { Set failingTaskIndices = Sets.newHashSet(); int failingTaskAttemptUpto = 0; Integer failAll = new Integer(-1); - + int verifyValue = -1; Set verifyTaskIndices = Sets.newHashSet(); - + /** * Enable failure for this processor */ @@ -76,7 +76,7 @@ public class TestProcessor extends AbstractLogicalIOProcessor { * Probability to random fail a task attempt. Range is 0 to 1. The number is set per DAG. */ public static String TEZ_FAILING_PROCESSOR_RANDOM_FAIL_PROBABILITY = "tez.failing-processor.random-fail-probability"; - + /** * Time to sleep in the processor in milliseconds. */ @@ -94,10 +94,10 @@ public class TestProcessor extends AbstractLogicalIOProcessor { */ public static String TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT = "tez.failing-processor.failing-upto-task-attempt"; - - public static String TEZ_FAILING_PROCESSOR_VERIFY_VALUE = + + public static String TEZ_FAILING_PROCESSOR_VERIFY_VALUE = "tez.failing-processor.verify-value"; - + public static String TEZ_FAILING_PROCESSOR_VERIFY_TASK_INDEX = "tez.failing-processor.verify-task-index"; @@ -120,19 +120,19 @@ public static ProcessorDescriptor getProcDesc(UserPayload payload) { void throwException(String msg) { RuntimeException e = new RuntimeException(msg); - getContext().reportFailure(TaskFailureType.NON_FATAL, e , msg); + getContext().reportFailure(TaskFailureType.NON_FATAL, e, msg); throw e; } public static String getVertexConfName(String confName, String vertexName) { return confName + "." + vertexName; } - + public static String getVertexConfName(String confName, String vertexName, - int taskIndex) { + int taskIndex) { return confName + "." + vertexName + "." + String.valueOf(taskIndex); } - + @Override public void initialize() throws Exception { if (getContext().getUserPayload() != null && getContext().getUserPayload().hasPayload()) { @@ -164,10 +164,10 @@ public void initialize() throws Exception { } failingTaskAttemptUpto = conf.getInt( getVertexConfName(TEZ_FAILING_PROCESSOR_FAILING_UPTO_TASK_ATTEMPT, vName), 0); - LOG.info("Adding failing attempt : " + failingTaskAttemptUpto + + LOG.info("Adding failing attempt : " + failingTaskAttemptUpto + " dag: " + getContext().getDAGName()); } - + doRandomFail = conf .getBoolean(TEZ_FAILING_PROCESSOR_DO_RANDOM_FAIL, false); randomFailProbability = conf.getFloat(TEZ_FAILING_PROCESSOR_RANDOM_FAIL_PROBABILITY, 0.0f); @@ -186,7 +186,7 @@ public void close() throws Exception { @Override public void run(Map inputs, - Map outputs) throws Exception { + Map outputs) throws Exception { LOG.info("Sleeping ms: " + sleepMs); for (LogicalInput input : inputs.values()) { @@ -197,16 +197,16 @@ public void run(Map inputs, } Thread.sleep(sleepMs); - + if (!doRandomFail) { // not random fail if (doFail) { if ( (failingTaskIndices.contains(failAll) || - failingTaskIndices.contains(getContext().getTaskIndex())) && - (failingTaskAttemptUpto == failAll.intValue() || - failingTaskAttemptUpto >= getContext().getTaskAttemptNumber())) { - String msg = "FailingProcessor: " + getContext().getUniqueIdentifier() + + failingTaskIndices.contains(getContext().getTaskIndex())) && + (failingTaskAttemptUpto == failAll.intValue() || + failingTaskAttemptUpto >= getContext().getTaskAttemptNumber())) { + String msg = "FailingProcessor: " + getContext().getUniqueIdentifier() + " dag: " + getContext().getDAGName() + " taskIndex: " + getContext().getTaskIndex() + " taskAttempt: " + getContext().getTaskAttemptNumber(); @@ -218,17 +218,17 @@ public void run(Map inputs, // random fail // If task attempt number is below limit, try to randomly fail the attempt. int taskAttemptNumber = getContext().getTaskAttemptNumber(); - int maxFailedAttempt = conf.getInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, - TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS_DEFAULT); + int maxFailedAttempt = conf.getInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, + TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS_DEFAULT); if (taskAttemptNumber < maxFailedAttempt - 1) { float rollNumber = (float) Math.random(); - String msg = "FailingProcessor random fail turned on." + - " Do a roll: " + getContext().getUniqueIdentifier() + + String msg = "FailingProcessor random fail turned on." + + " Do a roll: " + getContext().getUniqueIdentifier() + " dag: " + getContext().getDAGName() + " taskIndex: " + getContext().getTaskIndex() + " taskAttempt: " + taskAttemptNumber + " maxFailedAttempt: " + maxFailedAttempt + - " rollNumber: " + rollNumber + + " rollNumber: " + rollNumber + " randomFailProbability " + randomFailProbability; LOG.info(msg); if (rollNumber < randomFailProbability) { @@ -239,21 +239,21 @@ public void run(Map inputs, } } } - + if (inputs.entrySet().size() > 0) { - String msg = "Reading input of current FailingProcessor: " + getContext().getUniqueIdentifier() + - " dag: " + getContext().getDAGName() + - " vertex: " + getContext().getTaskVertexName() + - " taskIndex: " + getContext().getTaskIndex() + - " taskAttempt: " + getContext().getTaskAttemptNumber(); - LOG.info(msg); + String msg = "Reading input of current FailingProcessor: " + getContext().getUniqueIdentifier() + + " dag: " + getContext().getDAGName() + + " vertex: " + getContext().getTaskVertexName() + + " taskIndex: " + getContext().getTaskIndex() + + " taskAttempt: " + getContext().getTaskAttemptNumber(); + LOG.info(msg); } //initialize sum to attempt number + 1 int sum = getContext().getTaskAttemptNumber() + 1; LOG.info("initializing vertex= " + getContext().getTaskVertexName() + - " taskIndex: " + getContext().getTaskIndex() + - " taskAttempt: " + getContext().getTaskAttemptNumber() + - " sum= " + sum); + " taskIndex: " + getContext().getTaskIndex() + + " taskAttempt: " + getContext().getTaskAttemptNumber() + + " sum= " + sum); //sum = summation of input values for (Map.Entry entry : inputs.entrySet()) { if (!(entry.getValue() instanceof TestInput)) { @@ -266,14 +266,14 @@ public void run(Map inputs, LOG.info("Reading input: " + entry.getKey() + " inputValue= " + inputValue); sum += inputValue; } - + if (outputs.entrySet().size() > 0) { - String msg = "Writing output of current FailingProcessor: " + getContext().getUniqueIdentifier() + - " dag: " + getContext().getDAGName() + - " vertex: " + getContext().getTaskVertexName() + - " taskIndex: " + getContext().getTaskIndex() + - " taskAttempt: " + getContext().getTaskAttemptNumber(); - LOG.info(msg); + String msg = "Writing output of current FailingProcessor: " + getContext().getUniqueIdentifier() + + " dag: " + getContext().getDAGName() + + " vertex: " + getContext().getTaskVertexName() + + " taskIndex: " + getContext().getTaskIndex() + + " taskAttempt: " + getContext().getTaskAttemptNumber(); + LOG.info(msg); } for (Map.Entry entry : outputs.entrySet()) { if (!(entry.getValue() instanceof TestOutput)) { @@ -285,8 +285,8 @@ public void run(Map inputs, TestOutput output = (TestOutput) entry.getValue(); output.write(sum); } - - LOG.info("Output for DAG: " + getContext().getDAGName() + + LOG.info("Output for DAG: " + getContext().getDAGName() + " vertex: " + getContext().getTaskVertexName() + " task: " + getContext().getTaskIndex() + " attempt: " + getContext().getTaskAttemptNumber() @@ -295,12 +295,12 @@ public void run(Map inputs, .contains(new Integer(getContext().getTaskIndex()))) { if (verifyValue != -1 && verifyValue != sum) { // expected output value set and not equal to observed value - String msg = "Expected output mismatch of current FailingProcessor: " - + getContext().getUniqueIdentifier() + - " dag: " + getContext().getDAGName() + - " vertex: " + getContext().getTaskVertexName() + - " taskIndex: " + getContext().getTaskIndex() + - " taskAttempt: " + getContext().getTaskAttemptNumber(); + String msg = "Expected output mismatch of current FailingProcessor: " + + getContext().getUniqueIdentifier() + + " dag: " + getContext().getDAGName() + + " vertex: " + getContext().getTaskVertexName() + + " taskIndex: " + getContext().getTaskIndex() + + " taskAttempt: " + getContext().getTaskAttemptNumber(); msg += "\n" + "Expected output: " + verifyValue + " got: " + sum; throwException(msg); } else { @@ -311,5 +311,4 @@ public void run(Map inputs, } } } - } diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestRecovery.java b/tez-tests/src/test/java/org/apache/tez/test/TestRecovery.java index c7b1fb9639..e2e2ce53fe 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestRecovery.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestRecovery.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -140,7 +140,7 @@ public static void afterClass() throws InterruptedException { } } - @Test(timeout=1800000) + @Test(timeout = 1800000) public void testRecovery_OrderedWordCount() throws Exception { ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1); @@ -151,9 +151,9 @@ public void testRecovery_OrderedWordCount() throws Exception { ContainerId containerId = ContainerId.newInstance( ApplicationAttemptId.newInstance(appId, 1), 1); NodeId nodeId = NodeId.newInstance("localhost", 10); - + List initGeneratedEvents = Lists.newArrayList( - new TezEvent(InputDataInformationEvent.createWithObjectPayload(0, new Object()), null)); + new TezEvent(InputDataInformationEvent.createWithObjectPayload(0, new Object()), null)); List shutdownConditions = Lists .newArrayList( @@ -178,7 +178,7 @@ public void testRecovery_OrderedWordCount() throws Exception { new SimpleShutdownCondition(TIMING.POST, new VertexConfigurationDoneEvent(vertexId0, 0L, 2, null, null, null, true)), - + new SimpleShutdownCondition(TIMING.POST, new VertexConfigurationDoneEvent(vertexId1, 0L, 2, null, null, null, true)), @@ -251,7 +251,7 @@ public void testRecovery_OrderedWordCount() throws Exception { } private void testOrderedWordCount(SimpleShutdownCondition shutdownCondition, - boolean enableAutoParallelism, boolean generateSplitInClient) throws Exception { + boolean enableAutoParallelism, boolean generateSplitInClient) throws Exception { LOG.info("shutdownCondition:" + shutdownCondition.getEventType() + ", event=" + shutdownCondition.getEvent()); String inputDirStr = "/tmp/owc-input/"; @@ -300,17 +300,16 @@ private void testOrderedWordCount(SimpleShutdownCondition shutdownCondition, assertEquals(shutdownCondition.getEvent().getEventType(), lastEvent.getEventType()); assertTrue(shutdownCondition.match(lastEvent)); - } private void testOrderedWordCountMultipleRoundRecoverying( - RecoveryServiceWithEventHandlingHook.MultipleRoundShutdownCondition shutdownCondition, - boolean enableAutoParallelism, boolean generateSplitInClient) throws Exception { + RecoveryServiceWithEventHandlingHook.MultipleRoundShutdownCondition shutdownCondition, + boolean enableAutoParallelism, boolean generateSplitInClient) throws Exception { - for (int i=0; i(), ApplicationAttemptId - .newInstance(appId, 1), null)), + .newInstance(appId, 1), null)), new SimpleShutdownCondition(TIMING.POST, new VertexInitializedEvent( vertexId0, "hashSide", 0L, 0L, 0, "", null, initGeneratedEvents, null)), new SimpleShutdownCondition(TIMING.POST, new VertexInitializedEvent( @@ -394,7 +393,7 @@ public void testRecovery_HashJoin() throws Exception { new SimpleShutdownCondition(TIMING.POST, new VertexConfigurationDoneEvent(vertexId0, 0L, 2, null, null, null, true)), - + new SimpleShutdownCondition(TIMING.POST, new VertexConfigurationDoneEvent(vertexId1, 0L, 2, null, null, null, true)), @@ -402,7 +401,7 @@ public void testRecovery_HashJoin() throws Exception { new SimpleShutdownCondition(TIMING.POST, new VertexConfigurationDoneEvent(vertexId2, 0L, 2, null, null, null, true)), - + new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent( vertexId0, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, VertexState.SUCCEEDED, "", new TezCounters(), new VertexStats(), @@ -461,7 +460,7 @@ public void testRecovery_HashJoin() throws Exception { } private void testHashJoinExample(SimpleShutdownCondition shutdownCondition, - boolean enableAutoParallelism, boolean generateSplitInClient) throws Exception { + boolean enableAutoParallelism, boolean generateSplitInClient) throws Exception { HashJoinExample hashJoinExample = new HashJoinExample(); TezConfiguration tezConf = new TezConfiguration(miniTezCluster.getConfig()); tezConf.setInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS, 4); @@ -479,9 +478,9 @@ private void testHashJoinExample(SimpleShutdownCondition shutdownCondition, RecoveryService.TEZ_TEST_RECOVERY_DRAIN_EVENTS_WHEN_STOPPED, false); tezConf.setBoolean( TezConfiguration.TEZ_AM_STAGING_SCRATCH_DATA_AUTO_DELETE, false); - tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,0); + tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0); tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0); - tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY,1000); + tezConf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY, 1000); tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "INFO;org.apache.tez=DEBUG"); hashJoinExample.setConf(tezConf); @@ -559,57 +558,56 @@ public boolean accept(Path p) { @Test(timeout = 1800000) public void testTwoRoundsRecoverying() throws Exception { ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), - 1); + 1); TezDAGID dagId = TezDAGID.getInstance(appId, 1); TezVertexID vertexId0 = TezVertexID.getInstance(dagId, 0); TezVertexID vertexId1 = TezVertexID.getInstance(dagId, 1); TezVertexID vertexId2 = TezVertexID.getInstance(dagId, 2); ContainerId containerId = ContainerId.newInstance( - ApplicationAttemptId.newInstance(appId, 1), 1); + ApplicationAttemptId.newInstance(appId, 1), 1); NodeId nodeId = NodeId.newInstance("localhost", 10); List initGeneratedEvents = Lists.newArrayList( - new TezEvent(InputDataInformationEvent.createWithObjectPayload(0, new Object()), null)); - + new TezEvent(InputDataInformationEvent.createWithObjectPayload(0, new Object()), null)); List shutdownConditions = Lists.newArrayList( - new SimpleShutdownCondition(TIMING.POST, new DAGInitializedEvent( - dagId, 0L, "username", "dagName", null)), - new SimpleShutdownCondition(TIMING.POST, new DAGStartedEvent(dagId, - 0L, "username", "dagName")), - new SimpleShutdownCondition(TIMING.POST, - new VertexInitializedEvent(vertexId0, "Tokenizer", 0L, 0L, 0, - "", null, initGeneratedEvents, null)), - new SimpleShutdownCondition(TIMING.POST, new VertexStartedEvent( - vertexId0, 0L, 0L)), - new SimpleShutdownCondition(TIMING.POST, - new VertexConfigurationDoneEvent(vertexId0, 0L, 2, null, null, - null, true)), - new SimpleShutdownCondition(TIMING.POST, new TaskStartedEvent( - TezTaskID.getInstance(vertexId0, 0), "vertexName", 0L, 0L)), - new SimpleShutdownCondition(TIMING.POST, - new TaskAttemptStartedEvent(TezTaskAttemptID.getInstance( - TezTaskID.getInstance(vertexId0, 0), 0), "vertexName", 0L, - containerId, nodeId, "", "", "")), - new SimpleShutdownCondition(TIMING.POST, new TaskFinishedEvent( - TezTaskID.getInstance(vertexId0, 0), "vertexName", 0L, 0L, - null, TaskState.SUCCEEDED, "", new TezCounters(), 0)), - new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent( - vertexId0, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, - VertexState.SUCCEEDED, "", new TezCounters(), - new VertexStats(), new HashMap(), null)), - new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent( - vertexId1, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, - VertexState.SUCCEEDED, "", new TezCounters(), - new VertexStats(), new HashMap(), null)), - new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent( - vertexId2, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, - VertexState.SUCCEEDED, "", new TezCounters(), - new VertexStats(), new HashMap(), null)), - new SimpleShutdownCondition(TIMING.POST, new DAGFinishedEvent( - dagId, 0L, 0L, DAGState.SUCCEEDED, "", new TezCounters(), - "username", "dagName", new HashMap(), - ApplicationAttemptId.newInstance(appId, 1), null)) + new SimpleShutdownCondition(TIMING.POST, new DAGInitializedEvent( + dagId, 0L, "username", "dagName", null)), + new SimpleShutdownCondition(TIMING.POST, new DAGStartedEvent(dagId, + 0L, "username", "dagName")), + new SimpleShutdownCondition(TIMING.POST, + new VertexInitializedEvent(vertexId0, "Tokenizer", 0L, 0L, 0, + "", null, initGeneratedEvents, null)), + new SimpleShutdownCondition(TIMING.POST, new VertexStartedEvent( + vertexId0, 0L, 0L)), + new SimpleShutdownCondition(TIMING.POST, + new VertexConfigurationDoneEvent(vertexId0, 0L, 2, null, null, + null, true)), + new SimpleShutdownCondition(TIMING.POST, new TaskStartedEvent( + TezTaskID.getInstance(vertexId0, 0), "vertexName", 0L, 0L)), + new SimpleShutdownCondition(TIMING.POST, + new TaskAttemptStartedEvent(TezTaskAttemptID.getInstance( + TezTaskID.getInstance(vertexId0, 0), 0), "vertexName", 0L, + containerId, nodeId, "", "", "")), + new SimpleShutdownCondition(TIMING.POST, new TaskFinishedEvent( + TezTaskID.getInstance(vertexId0, 0), "vertexName", 0L, 0L, + null, TaskState.SUCCEEDED, "", new TezCounters(), 0)), + new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent( + vertexId0, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, + VertexState.SUCCEEDED, "", new TezCounters(), + new VertexStats(), new HashMap(), null)), + new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent( + vertexId1, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, + VertexState.SUCCEEDED, "", new TezCounters(), + new VertexStats(), new HashMap(), null)), + new SimpleShutdownCondition(TIMING.POST, new VertexFinishedEvent( + vertexId2, "vertexName", 1, 0L, 0L, 0L, 0L, 0L, + VertexState.SUCCEEDED, "", new TezCounters(), + new VertexStats(), new HashMap(), null)), + new SimpleShutdownCondition(TIMING.POST, new DAGFinishedEvent( + dagId, 0L, 0L, DAGState.SUCCEEDED, "", new TezCounters(), + "username", "dagName", new HashMap(), + ApplicationAttemptId.newInstance(appId, 1), null)) ); @@ -617,14 +615,14 @@ public void testTwoRoundsRecoverying() throws Exception { for (int i = 0; i < shutdownConditions.size() - 1; i++) { // randomly choose half of the test scenario to avoid // timeout. - if (rand.nextDouble()<0.5) { + if (rand.nextDouble() < 0.5) { int nextSimpleConditionIndex = i + 1 + rand.nextInt(shutdownConditions.size() - i - 1); if (nextSimpleConditionIndex == shutdownConditions.size() - 1) { testOrderedWordCountMultipleRoundRecoverying( - new RecoveryServiceWithEventHandlingHook.MultipleRoundShutdownCondition( - Lists.newArrayList(shutdownConditions.get(i), shutdownConditions.get(nextSimpleConditionIndex))) - , true, - shutdownConditions.get(i).getHistoryEvent().getEventType() == HistoryEventType.VERTEX_STARTED); + new RecoveryServiceWithEventHandlingHook.MultipleRoundShutdownCondition( + Lists.newArrayList(shutdownConditions.get(i), shutdownConditions.get(nextSimpleConditionIndex))) + , true, + shutdownConditions.get(i).getHistoryEvent().getEventType() == HistoryEventType.VERTEX_STARTED); } } } diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java b/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java index f9d4469779..db1a6e8441 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestSecureShuffle.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -85,7 +85,7 @@ public class TestSecureShuffle { private boolean asyncHttp; public TestSecureShuffle(boolean sslInCluster, int resultWithTezSSL, int resultWithoutTezSSL, - boolean asyncHttp) { + boolean asyncHttp) { this.enableSSLInCluster = sslInCluster; this.resultWithTezSSL = resultWithTezSSL; this.resultWithoutTezSSL = resultWithoutTezSSL; @@ -97,15 +97,15 @@ public TestSecureShuffle(boolean sslInCluster, int resultWithTezSSL, int resultW public static Collection getParameters() { Collection parameters = new ArrayList(); //enable ssl in cluster, succeed with tez-ssl enabled, fail with tez-ssl disabled - parameters.add(new Object[] { true, 0, 1, false }); + parameters.add(new Object[]{true, 0, 1, false}); //With asyncHttp - parameters.add(new Object[] { true, 0, 1, true }); - parameters.add(new Object[] { false, 1, 0, true }); + parameters.add(new Object[]{true, 0, 1, true}); + parameters.add(new Object[]{false, 1, 0, true}); //Negative testcase //disable ssl in cluster, fail with tez-ssl enabled, succeed with tez-ssl disabled - parameters.add(new Object[] { false, 1, 0, false }); + parameters.add(new Object[]{false, 1, 0, false}); return parameters; } @@ -174,8 +174,8 @@ private void baseTest(int expectedResult) throws Exception { TestOrderedWordCount wordCount = new TestOrderedWordCount(); wordCount.setConf(new Configuration(miniTezCluster.getConfig())); - String[] args = new String[] { "-DUSE_MR_CONFIGS=false", - inputLoc.toString(), outputLoc.toString() }; + String[] args = new String[]{"-DUSE_MR_CONFIGS=false", + inputLoc.toString(), outputLoc.toString()}; assertEquals(expectedResult, wordCount.run(args)); } @@ -239,7 +239,7 @@ private static void setupKeyStores() throws Exception { * which fixes another issue. */ public static void setupSSLConfig(String keystoresDir, String sslConfDir, Configuration config, - boolean useClientCert, boolean trustStore, String excludeCiphers) throws Exception { + boolean useClientCert, boolean trustStore, String excludeCiphers) throws Exception { String clientKS = keystoresDir + "/clientKS.jks"; String clientPassword = "clientP"; String serverKS = keystoresDir + "/serverKS.jks"; @@ -263,7 +263,7 @@ public static void setupSSLConfig(String keystoresDir, String sslConfDir, Config String localhostName = InetAddress.getLocalHost().getHostName(); KeyPair sKP = KeyStoreTestUtil.generateKeyPair("RSA"); X509Certificate sCert = - generateCertificate("CN="+localhostName+", O=server", sKP, 30, "SHA1withRSA"); + generateCertificate("CN=" + localhostName + ", O=server", sKP, 30, "SHA1withRSA"); KeyStoreTestUtil.createKeyStore(serverKS, serverPassword, "server", sKP.getPrivate(), sCert); certs.put("server", sCert); @@ -305,7 +305,7 @@ public static X509Certificate generateCertificate(String dn, KeyPair pair, int d String hostName = InetAddress.getLocalHost().getHostName(); String hostAddress = InetAddress.getLocalHost().getHostAddress(); certGen.addExtension(X509Extensions.SubjectAlternativeName, false, - new GeneralNames(new GeneralName[] { + new GeneralNames(new GeneralName[]{ new GeneralName(GeneralName.iPAddress, hostAddress), new GeneralName(GeneralName.dNSName, hostName), new GeneralName(GeneralName.dNSName, "localhost") diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestTaskErrorsUsingLocalMode.java b/tez-tests/src/test/java/org/apache/tez/test/TestTaskErrorsUsingLocalMode.java index d7d2dd925f..aa8b1efe7a 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestTaskErrorsUsingLocalMode.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestTaskErrorsUsingLocalMode.java @@ -48,7 +48,6 @@ public class TestTaskErrorsUsingLocalMode { private static final File STAGING_DIR = new File(System.getProperty("test.build.data"), TestTaskErrorsUsingLocalMode.class.getName()).getAbsoluteFile(); - @Test(timeout = 20000) public void testFatalErrorReported() throws IOException, TezException, InterruptedException { @@ -121,7 +120,6 @@ public void testSelfKillReported() throws IOException, TezException, Interrupted } } - private TezClient getTezClient(String name) throws IOException, TezException { TezConfiguration tezConf1 = new TezConfiguration(); tezConf1.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true); @@ -134,7 +132,6 @@ private TezClient getTezClient(String name) throws IOException, TezException { return tezClient1; } - public static class FailingProcessor extends AbstractLogicalIOProcessor { private static final String FAIL_STRING_NON_FATAL = "non-fatal-fail"; @@ -147,7 +144,6 @@ public static class FailingProcessor extends AbstractLogicalIOProcessor { private static volatile boolean shouldKill; private static volatile int killModeAttemptNumberToSucceed; - static { reset(); } @@ -216,5 +212,4 @@ public void run(Map inputs, Map out } } } - } diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java b/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java index 6de272358b..45781bce82 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java +++ b/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -106,10 +106,9 @@ import org.junit.BeforeClass; import org.junit.Test; - /** * Tests for Tez example jobs - * + * */ public class TestTezJobs { @@ -144,7 +143,6 @@ public static void setup() throws IOException { mrrTezCluster.init(conf); mrrTezCluster.start(); } - } @AfterClass @@ -193,9 +191,9 @@ public void testHashJoinExample() throws Exception { out1.close(); out2.close(); - String[] args = new String[] { + String[] args = new String[]{ "-D" + TezConfiguration.TEZ_AM_STAGING_DIR + "=" + stagingDirPath.toString(), - "-counter", inPath1.toString(), inPath2.toString(), "1", outPath.toString() }; + "-counter", inPath1.toString(), inPath2.toString(), "1", outPath.toString()}; assertEquals(0, hashJoinExample.run(args)); FileStatus[] statuses = remoteFs.listStatus(outPath, new PathFilter() { @@ -246,22 +244,22 @@ public void testHashJoinExampleWithDataViaEvent() throws Exception { tezSession.start(); JoinDataGen dataGen = new JoinDataGen(); - String[] dataGenArgs = new String[] { - "-counter", - dataPath1.toString(), "1048576", dataPath2.toString(), "8", - expectedOutputPath.toString(), "2" }; + String[] dataGenArgs = new String[]{ + "-counter", + dataPath1.toString(), "1048576", dataPath2.toString(), "8", + expectedOutputPath.toString(), "2"}; assertEquals(0, dataGen.run(tezConf, dataGenArgs, tezSession)); HashJoinExample joinExample = new HashJoinExample(); - String[] args = new String[] { - dataPath1.toString(), dataPath2.toString(), "1", outPath.toString(), - "doBroadcast"}; + String[] args = new String[]{ + dataPath1.toString(), dataPath2.toString(), "1", outPath.toString(), + "doBroadcast"}; assertEquals(0, joinExample.run(tezConf, args, tezSession)); JoinValidate joinValidate = new JoinValidate(); - String[] validateArgs = new String[] { - "-counter", expectedOutputPath.toString(), outPath.toString(), "3" }; + String[] validateArgs = new String[]{ + "-counter", expectedOutputPath.toString(), outPath.toString(), "3"}; assertEquals(0, joinValidate.run(tezConf, validateArgs, tezSession)); } finally { if (tezSession != null) { @@ -304,10 +302,10 @@ public void testHashJoinExampleDisableSplitGrouping() throws Exception { out1.close(); out2.close(); - String[] args = new String[] { + String[] args = new String[]{ "-D" + TezConfiguration.TEZ_AM_STAGING_DIR + "=" + stagingDirPath.toString(), "-counter", "-local", "-disableSplitGrouping", - inPath1.toString(), inPath2.toString(), "1", outPath.toString() }; + inPath1.toString(), inPath2.toString(), "1", outPath.toString()}; assertEquals(0, hashJoinExample.run(args)); FileStatus[] statuses = localFs.listStatus(outPath, new PathFilter() { @@ -361,10 +359,10 @@ public void testSortMergeJoinExample() throws Exception { out1.close(); out2.close(); - String[] args = new String[] { + String[] args = new String[]{ "-D" + TezConfiguration.TEZ_AM_STAGING_DIR + "=" + stagingDirPath.toString(), "-D" + TezConfiguration.TEZ_AM_APPLICATION_PRIORITY + "=" + "2", - "-counter", inPath1.toString(), inPath2.toString(), "1", outPath.toString() }; + "-counter", inPath1.toString(), inPath2.toString(), "1", outPath.toString()}; assertEquals(0, sortMergeJoinExample.run(args)); FileStatus[] statuses = remoteFs.listStatus(outPath, new PathFilter() { @@ -403,9 +401,9 @@ public void testPerIOCounterAggregation() throws Exception { SortMergeJoinHelper sortMergeJoinHelper = new SortMergeJoinHelper(tezClient); sortMergeJoinHelper.setConf(conf); - String[] args = new String[] { + String[] args = new String[]{ "-D" + TezConfiguration.TEZ_AM_STAGING_DIR + "=" + stagingDirPath.toString(), - "-counter", inPath1.toString(), inPath2.toString(), "1", outPath.toString() }; + "-counter", inPath1.toString(), inPath2.toString(), "1", outPath.toString()}; assertEquals(0, sortMergeJoinHelper.run(conf, args, tezClient)); verifySortMergeJoinInput(outPath, expectedResults); @@ -450,7 +448,7 @@ public void testPerIOCounterAggregation() throws Exception { int nonZeroCounters = 0; // verify that the sum of the counter values for edges add up to the aggregated counter value. - for(TaskCounter c : countersToVerifyAgg) { + for (TaskCounter c : countersToVerifyAgg) { TezCounter aggregatedCounter = aggregatedGroup.findCounter(c.name(), false); TezCounter input1Counter = input1Group.findCounter(c.name(), false); TezCounter input2Counter = input2Group.findCounter(c.name(), false); @@ -481,7 +479,6 @@ public void testPerIOCounterAggregation() throws Exception { aggregateCounter.getValue(), joinerOutputCounter.getValue()); } - @Test(timeout = 60000) public void testSortMergeJoinExampleDisableSplitGrouping() throws Exception { SortMergeJoinExample sortMergeJoinExample = new SortMergeJoinExample(); @@ -516,10 +513,10 @@ public void testSortMergeJoinExampleDisableSplitGrouping() throws Exception { out1.close(); out2.close(); - String[] args = new String[] { + String[] args = new String[]{ "-D" + TezConfiguration.TEZ_AM_STAGING_DIR + "=" + stagingDirPath.toString(), - "-counter", "-local","-disableSplitGrouping", - inPath1.toString(), inPath2.toString(), "1", outPath.toString() }; + "-counter", "-local", "-disableSplitGrouping", + inPath1.toString(), inPath2.toString(), "1", outPath.toString()}; assertEquals(0, sortMergeJoinExample.run(args)); FileStatus[] statuses = localFs.listStatus(outPath, new PathFilter() { @@ -566,22 +563,21 @@ public void testHashJoinExamplePipeline() throws Exception { tezSession.start(); JoinDataGen dataGen = new JoinDataGen(); - String[] dataGenArgs = new String[] { + String[] dataGenArgs = new String[]{ "-counter", dataPath1.toString(), "1048576", dataPath2.toString(), "524288", - expectedOutputPath.toString(), "2" }; + expectedOutputPath.toString(), "2"}; assertEquals(0, dataGen.run(tezConf, dataGenArgs, tezSession)); HashJoinExample joinExample = new HashJoinExample(); - String[] args = new String[] { - dataPath1.toString(), dataPath2.toString(), "2", outPath.toString() }; + String[] args = new String[]{ + dataPath1.toString(), dataPath2.toString(), "2", outPath.toString()}; assertEquals(0, joinExample.run(tezConf, args, tezSession)); JoinValidate joinValidate = new JoinValidate(); - String[] validateArgs = new String[] { - "-counter", expectedOutputPath.toString(), outPath.toString(), "3" }; + String[] validateArgs = new String[]{ + "-counter", expectedOutputPath.toString(), outPath.toString(), "3"}; assertEquals(0, joinValidate.run(tezConf, validateArgs, tezSession)); - } finally { if (tezSession != null) { tezSession.stop(); @@ -615,21 +611,20 @@ public void testSortMergeJoinExamplePipeline() throws Exception { tezSession.start(); JoinDataGen dataGen = new JoinDataGen(); - String[] dataGenArgs = new String[] { + String[] dataGenArgs = new String[]{ dataPath1.toString(), "1048576", dataPath2.toString(), "524288", - expectedOutputPath.toString(), "2" }; + expectedOutputPath.toString(), "2"}; assertEquals(0, dataGen.run(tezConf, dataGenArgs, tezSession)); SortMergeJoinExample joinExample = new SortMergeJoinExample(); - String[] args = new String[] { - dataPath1.toString(), dataPath2.toString(), "2", outPath.toString() }; + String[] args = new String[]{ + dataPath1.toString(), dataPath2.toString(), "2", outPath.toString()}; assertEquals(0, joinExample.run(tezConf, args, tezSession)); JoinValidate joinValidate = new JoinValidate(); - String[] validateArgs = new String[] { - expectedOutputPath.toString(), outPath.toString(), "3" }; + String[] validateArgs = new String[]{ + expectedOutputPath.toString(), outPath.toString(), "3"}; assertEquals(0, joinValidate.run(tezConf, validateArgs, tezSession)); - } finally { if (tezSession != null) { tezSession.stop(); @@ -685,16 +680,16 @@ public static void verifyOrderedWordCountOutput(Path resultFile, FileSystem fs) while ((line = reader.readLine()) != null) { LOG.info("Line: " + line + ", counter=" + currentCounter); int pos = line.indexOf("\t"); - String word = line.substring(0, pos-1); + String word = line.substring(0, pos - 1); Assert.assertEquals(prefix + "_" + currentCounter, word); - String val = line.substring(pos+1, line.length()); - Assert.assertEquals((long)(11 - currentCounter) * 2, (long)Long.valueOf(val)); + String val = line.substring(pos + 1, line.length()); + Assert.assertEquals((long) (11 - currentCounter) * 2, (long) Long.valueOf(val)); currentCounter--; } Assert.assertEquals(0, currentCounter); } - + public static void verifyOutput(Path outputDir, FileSystem fs) throws IOException { FileStatus[] fileStatuses = fs.listStatus(outputDir); Path resultFile = null; @@ -723,7 +718,7 @@ public static void verifyOutput(Path outputDir, FileSystem fs) throws IOExceptio assertTrue(foundSuccessFile); verifyOrderedWordCountOutput(resultFile, fs); } - + @Test(timeout = 60000) public void testOrderedWordCount() throws Exception { String inputDirStr = "/tmp/owc-input/"; @@ -743,18 +738,16 @@ public void testOrderedWordCount() throws Exception { try { OrderedWordCount job = new OrderedWordCount(); - Assert.assertTrue("OrderedWordCount failed", job.run(tezConf, new String[]{"-counter", inputDirStr, outputDirStr, "2"}, null)==0); + Assert.assertTrue("OrderedWordCount failed", job.run(tezConf, new String[]{"-counter", inputDirStr, outputDirStr, "2"}, null) == 0); verifyOutput(outputDir, remoteFs); - } finally { remoteFs.delete(stagingDirPath, true); if (tezSession != null) { tezSession.stop(); } } - } - + @Test(timeout = 60000) public void testOrderedWordCountDisableSplitGrouping() throws Exception { String inputDirStr = TEST_ROOT_DIR + "/tmp/owc-input/"; @@ -776,16 +769,14 @@ public void testOrderedWordCountDisableSplitGrouping() throws Exception { OrderedWordCount job = new OrderedWordCount(); Assert.assertTrue("OrderedWordCount failed", job.run(tezConf, new String[]{"-counter", "-local", "-disableSplitGrouping", - inputDirStr, outputDirStr, "2"}, null)==0); + inputDirStr, outputDirStr, "2"}, null) == 0); verifyOutput(outputDir, localFs); - } finally { localFs.delete(stagingDirPath, true); if (tezSession != null) { tezSession.stop(); } } - } @Test(timeout = 60000) @@ -797,28 +788,27 @@ public void testSimpleSessionExample() throws Exception { String[] inputPaths = new String[numIterations]; String[] outputPaths = new String[numIterations]; Path[] outputDirs = new Path[numIterations]; - for (int i=0; i apps = yarnClient.getApplications(); int appsBeforeCount = apps != null ? apps.size() : 0; @@ -826,16 +816,16 @@ public void testSimpleSessionExample() throws Exception { tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true); Assert.assertTrue( "SimpleSessionExample failed", - job.run(tezConf, new String[] { StringUtils.join(",", inputPaths), - StringUtils.join(",", outputPaths), "2" }, null) == 0); + job.run(tezConf, new String[]{StringUtils.join(",", inputPaths), + StringUtils.join(",", outputPaths), "2"}, null) == 0); - for (int i=0; i resultVertices = dagStatus.getVertexProgress().keySet(); - Assert.assertEquals(6, resultVertices.size()); - int i = 0; - for (String vertexName : resultVertices){ - if (i <= 1){ - Assert.assertTrue( vertexName.equals("v1") || vertexName.equals("v2")); - } else if (i == 2){ - Assert.assertTrue( vertexName.equals("v3")); - } else if (i <= 4){ - Assert.assertTrue( vertexName.equals("v4") || vertexName.equals("v5")); - } else { - Assert.assertTrue( vertexName.equals("v6")); + Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagStatus.getState()); + + // verify vertex order + Set resultVertices = dagStatus.getVertexProgress().keySet(); + Assert.assertEquals(6, resultVertices.size()); + int i = 0; + for (String vertexName : resultVertices) { + if (i <= 1) { + Assert.assertTrue(vertexName.equals("v1") || vertexName.equals("v2")); + } else if (i == 2) { + Assert.assertTrue(vertexName.equals("v3")); + } else if (i <= 4) { + Assert.assertTrue(vertexName.equals("v4") || vertexName.equals("v5")); + } else { + Assert.assertTrue(vertexName.equals("v6")); + } + i++; } - i++; - } } finally { if (tezClient != null) { tezClient.stop(); @@ -974,8 +961,8 @@ public void testInputInitializerEvents() throws TezException, InterruptedExcepti try { DAG dag = DAG.create("TestInputInitializerEvents"); Vertex vertex1 = Vertex.create(VERTEX_WITH_INITIALIZER_NAME, ProcessorDescriptor.create( - SleepProcessor.class.getName()) - .setUserPayload(new SleepProcessor.SleepProcessorConfig(1).toUserPayload()), 1) + SleepProcessor.class.getName()) + .setUserPayload(new SleepProcessor.SleepProcessorConfig(1).toUserPayload()), 1) .addDataSource(INPUT1_NAME, DataSourceDescriptor .create(InputDescriptor.create(MultiAttemptDAG.NoOpInput.class.getName()), @@ -1013,8 +1000,8 @@ public void testMultipleCommits_OnDAGSuccess() throws Exception { try { MultipleCommitsExample job = new MultipleCommitsExample(); Assert.assertTrue("MultipleCommitsExample failed", job.run(tezConf, - new String[]{ v1OutputPathPrefix, v1OutputNum + "", v2OutputPathPrefix, v2OutputNum + "", - uv12OutputPathPrefix, uv12OutputNum + "", v3OutputPathPrefix, v3OutputNum + ""}, null)==0); + new String[]{v1OutputPathPrefix, v1OutputNum + "", v2OutputPathPrefix, v2OutputNum + "", + uv12OutputPathPrefix, uv12OutputNum + "", v3OutputPathPrefix, v3OutputNum + ""}, null) == 0); verifyCommits(v1OutputPathPrefix, v1OutputNum); verifyCommits(v2OutputPathPrefix, v2OutputNum); verifyCommits(uv12OutputPathPrefix, uv12OutputNum); @@ -1026,7 +1013,7 @@ public void testMultipleCommits_OnDAGSuccess() throws Exception { } } } - + @Test(timeout = 60000) public void testMultipleCommits_OnVertexSuccess() throws Exception { Path stagingDirPath = new Path("/tmp/commit-staging-dir"); @@ -1046,9 +1033,9 @@ public void testMultipleCommits_OnVertexSuccess() throws Exception { try { MultipleCommitsExample job = new MultipleCommitsExample(); Assert.assertTrue("MultipleCommitsExample failed", job.run(tezConf, - new String[]{ v1OutputPathPrefix, v1OutputNum + "", v2OutputPathPrefix, v2OutputNum + "", - uv12OutputPathPrefix, uv12OutputNum + "", v3OutputPathPrefix, v3OutputNum + "", - MultipleCommitsExample.CommitOnVertexSuccessOption}, null)==0); + new String[]{v1OutputPathPrefix, v1OutputNum + "", v2OutputPathPrefix, v2OutputNum + "", + uv12OutputPathPrefix, uv12OutputNum + "", v3OutputPathPrefix, v3OutputNum + "", + MultipleCommitsExample.CommitOnVertexSuccessOption}, null) == 0); verifyCommits(v1OutputPathPrefix, v1OutputNum); verifyCommits(v2OutputPathPrefix, v2OutputNum); verifyCommits(uv12OutputPathPrefix, uv12OutputNum); @@ -1060,12 +1047,12 @@ public void testMultipleCommits_OnVertexSuccess() throws Exception { } } } - + private void verifyCommits(String outputPrefix, int outputNum) throws IllegalArgumentException, IOException { - for (int i=0; i< outputNum; ++i) { + for (int i = 0; i < outputNum; ++i) { String outputDir = outputPrefix + "_" + i; Assert.assertTrue("Output of " + outputDir + " is not succeeded", - remoteFs.exists(new Path( outputDir + "/_SUCCESS"))); + remoteFs.exists(new Path(outputDir + "/_SUCCESS"))); } } @@ -1134,7 +1121,6 @@ public List initialize() throws Exception { return null; } - @Override public void handleInputInitializerEvent(List events) throws Exception { lock.lock(); @@ -1294,7 +1280,6 @@ public void testAMClientHeartbeatTimeout() throws Exception { + ", report=" + report); Assert.assertTrue("Actual diagnostics: " + report.getDiagnostics(), report.getDiagnostics().contains("Client-to-AM Heartbeat timeout interval expired")); - } finally { remoteFs.delete(stagingDirPath, true); if (yarnClient != null) { @@ -1349,7 +1334,6 @@ public void testSessionTimeout() throws Exception { + ", report=" + report); Assert.assertTrue("Actual diagnostics: " + report.getDiagnostics(), report.getDiagnostics().contains("Session timed out")); - } finally { remoteFs.delete(stagingDirPath, true); if (yarnClient != null) { diff --git a/tez-tests/src/test/java/org/apache/tez/test/dag/MultiAttemptDAG.java b/tez-tests/src/test/java/org/apache/tez/test/dag/MultiAttemptDAG.java index cdf69e69b2..3764a87745 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/dag/MultiAttemptDAG.java +++ b/tez-tests/src/test/java/org/apache/tez/test/dag/MultiAttemptDAG.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -125,7 +125,7 @@ private synchronized void maybeScheduleTasks() { LOG.info("Scheduling tasks for vertex=" + getContext().getVertexName()); int numTasks = getContext().getVertexNumTasks(getContext().getVertexName()); List scheduledTasks = Lists.newArrayListWithCapacity(numTasks); - for (int i=0; i events) { + InputDescriptor inputDescriptor, List events) { List inputInfoEvents = new ArrayList(); - for (Event event: events) { + for (Event event : events) { if (event instanceof InputDataInformationEvent) { - inputInfoEvents.add((InputDataInformationEvent)event); + inputInfoEvents.add((InputDataInformationEvent) event); } } getContext().addRootInputEvents(inputName, inputInfoEvents); @@ -342,11 +342,8 @@ public List close() throws Exception { } } - - - public static DAG createDAG(String name, - Configuration conf) throws Exception { + Configuration conf) throws Exception { UserPayload payload = UserPayload.create(null); int taskCount = MULTI_ATTEMPT_DAG_VERTEX_NUM_TASKS_DEFAULT; if (conf != null) { @@ -360,13 +357,13 @@ public static DAG createDAG(String name, // Make each vertex manager fail on appropriate attempt v1.setVertexManagerPlugin(VertexManagerPluginDescriptor.create( - FailOnAttemptVertexManagerPlugin.class.getName()) + FailOnAttemptVertexManagerPlugin.class.getName()) .setUserPayload(UserPayload.create(ByteBuffer.wrap(new String("1").getBytes())))); v2.setVertexManagerPlugin(VertexManagerPluginDescriptor.create( - FailOnAttemptVertexManagerPlugin.class.getName()) + FailOnAttemptVertexManagerPlugin.class.getName()) .setUserPayload(UserPayload.create(ByteBuffer.wrap(new String("2").getBytes())))); v3.setVertexManagerPlugin(VertexManagerPluginDescriptor.create( - FailOnAttemptVertexManagerPlugin.class.getName()) + FailOnAttemptVertexManagerPlugin.class.getName()) .setUserPayload(UserPayload.create(ByteBuffer.wrap(new String("3").getBytes())))); dag.addVertex(v1).addVertex(v2).addVertex(v3); dag.addEdge(Edge.create(v1, v2, @@ -387,5 +384,4 @@ public static DAG createDAG(String name, public static DAG createDAG(Configuration conf) throws Exception { return createDAG("SimpleVTestDAG", conf); } - } diff --git a/tez-tests/src/test/java/org/apache/tez/test/dag/SimpleReverseVTestDAG.java b/tez-tests/src/test/java/org/apache/tez/test/dag/SimpleReverseVTestDAG.java index 4c8771cdbe..a0e8d7a7f4 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/dag/SimpleReverseVTestDAG.java +++ b/tez-tests/src/test/java/org/apache/tez/test/dag/SimpleReverseVTestDAG.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -45,9 +45,9 @@ public class SimpleReverseVTestDAG { public static String TEZ_SIMPLE_REVERSE_V_DAG_NUM_TASKS = "tez.simple-reverse-v-test-dag.num-tasks"; public static int TEZ_SIMPLE_REVERSE_V_DAG_NUM_TASKS_DEFAULT = 2; - - public static DAG createDAG(String name, - Configuration conf) throws Exception { + + public static DAG createDAG(String name, + Configuration conf) throws Exception { UserPayload payload = UserPayload.create(null); int taskCount = TEZ_SIMPLE_REVERSE_V_DAG_NUM_TASKS_DEFAULT; if (conf != null) { @@ -73,7 +73,7 @@ public static DAG createDAG(String name, TestInput.getInputDesc(payload)))); return dag; } - + public static DAG createDAG(Configuration conf) throws Exception { return createDAG("SimpleReverseVTestDAG", conf); } diff --git a/tez-tests/src/test/java/org/apache/tez/test/dag/SimpleVTestDAG.java b/tez-tests/src/test/java/org/apache/tez/test/dag/SimpleVTestDAG.java index a4eb95e059..ad38e7d30e 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/dag/SimpleVTestDAG.java +++ b/tez-tests/src/test/java/org/apache/tez/test/dag/SimpleVTestDAG.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -45,9 +45,9 @@ public class SimpleVTestDAG { public static String TEZ_SIMPLE_V_DAG_NUM_TASKS = "tez.simple-v-test-dag.num-tasks"; public static int TEZ_SIMPLE_V_DAG_NUM_TASKS_DEFAULT = 2; - - public static DAG createDAG(String name, - Configuration conf) throws Exception { + + public static DAG createDAG(String name, + Configuration conf) throws Exception { UserPayload payload = UserPayload.create(null); int taskCount = TEZ_SIMPLE_V_DAG_NUM_TASKS_DEFAULT; if (conf != null) { @@ -73,7 +73,7 @@ public static DAG createDAG(String name, TestInput.getInputDesc(payload)))); return dag; } - + public static DAG createDAG(Configuration conf) throws Exception { return createDAG("SimpleVTestDAG", conf); } diff --git a/tez-tests/src/test/java/org/apache/tez/test/dag/SixLevelsFailingDAG.java b/tez-tests/src/test/java/org/apache/tez/test/dag/SixLevelsFailingDAG.java index 036bedfdb4..194da173d7 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/dag/SixLevelsFailingDAG.java +++ b/tez-tests/src/test/java/org/apache/tez/test/dag/SixLevelsFailingDAG.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,6 +17,7 @@ */ package org.apache.tez.test.dag; + import org.apache.hadoop.conf.Configuration; import org.apache.tez.common.TezUtils; import org.apache.tez.dag.api.DAG; @@ -28,11 +29,11 @@ * A DAG with vertices divided into 6 levels. * Vertex name is "lv". Level/vertex numbers start at 1. * Each vertex has failing processor and failing inputs. The constructor can accept Tez Configuration to indicate failing patterns. - * + * * DAG is shown with a diagram below. * Each vertex has its degree of parallelism indicated in a bracket following its name. * Each edge annotates with data movement (s = scatter/gather, b = broadcast) - * + * * l1v1(1) l1v2(2) l1v3(3) l1v4(2) * |s |s |s |b * | | | | @@ -52,44 +53,44 @@ * */ public class SixLevelsFailingDAG extends ThreeLevelsFailingDAG { - - protected static Vertex l4v1; - protected static Vertex l5v1, l5v2, l5v3; - protected static Vertex l6v1; - - protected static void addDAGVerticesAndEdges() { - ThreeLevelsFailingDAG.addDAGVerticesAndEdges(); - l4v1 = Vertex.create("l4v1", TestProcessor.getProcDesc(payload), 10, defaultResource); - dag.addVertex(l4v1); - addEdge(l3v1, l4v1, DataMovementType.SCATTER_GATHER); - addEdge(l3v2, l4v1, DataMovementType.SCATTER_GATHER); - l5v1 = Vertex.create("l5v1", TestProcessor.getProcDesc(payload), 2, defaultResource); - dag.addVertex(l5v1); - addEdge(l4v1, l5v1, DataMovementType.SCATTER_GATHER); - l5v2 = Vertex.create("l5v2", TestProcessor.getProcDesc(payload), 4, defaultResource); - dag.addVertex(l5v2); - addEdge(l4v1, l5v2, DataMovementType.SCATTER_GATHER); - l5v3 = Vertex.create("l5v3", TestProcessor.getProcDesc(payload), 1, defaultResource); - dag.addVertex(l5v3); - addEdge(l4v1, l5v3, DataMovementType.SCATTER_GATHER); - l6v1 = Vertex.create("l6v1", TestProcessor.getProcDesc(payload), 4, defaultResource); - dag.addVertex(l6v1); - addEdge(l5v1, l6v1, DataMovementType.SCATTER_GATHER); - addEdge(l5v2, l6v1, DataMovementType.SCATTER_GATHER); - addEdge(l5v3, l6v1, DataMovementType.SCATTER_GATHER); - } - - public static DAG createDAG(String name, - Configuration conf) throws Exception { - if (conf != null) { - payload = TezUtils.createUserPayloadFromConf(conf); - } - dag = DAG.create(name); - addDAGVerticesAndEdges(); - return dag; - } - - public static DAG createDAG(Configuration conf) throws Exception { - return createDAG("SixLevelsFailingDAG", conf); + + protected static Vertex l4v1; + protected static Vertex l5v1, l5v2, l5v3; + protected static Vertex l6v1; + + protected static void addDAGVerticesAndEdges() { + ThreeLevelsFailingDAG.addDAGVerticesAndEdges(); + l4v1 = Vertex.create("l4v1", TestProcessor.getProcDesc(payload), 10, defaultResource); + dag.addVertex(l4v1); + addEdge(l3v1, l4v1, DataMovementType.SCATTER_GATHER); + addEdge(l3v2, l4v1, DataMovementType.SCATTER_GATHER); + l5v1 = Vertex.create("l5v1", TestProcessor.getProcDesc(payload), 2, defaultResource); + dag.addVertex(l5v1); + addEdge(l4v1, l5v1, DataMovementType.SCATTER_GATHER); + l5v2 = Vertex.create("l5v2", TestProcessor.getProcDesc(payload), 4, defaultResource); + dag.addVertex(l5v2); + addEdge(l4v1, l5v2, DataMovementType.SCATTER_GATHER); + l5v3 = Vertex.create("l5v3", TestProcessor.getProcDesc(payload), 1, defaultResource); + dag.addVertex(l5v3); + addEdge(l4v1, l5v3, DataMovementType.SCATTER_GATHER); + l6v1 = Vertex.create("l6v1", TestProcessor.getProcDesc(payload), 4, defaultResource); + dag.addVertex(l6v1); + addEdge(l5v1, l6v1, DataMovementType.SCATTER_GATHER); + addEdge(l5v2, l6v1, DataMovementType.SCATTER_GATHER); + addEdge(l5v3, l6v1, DataMovementType.SCATTER_GATHER); + } + + public static DAG createDAG(String name, + Configuration conf) throws Exception { + if (conf != null) { + payload = TezUtils.createUserPayloadFromConf(conf); } + dag = DAG.create(name); + addDAGVerticesAndEdges(); + return dag; + } + + public static DAG createDAG(Configuration conf) throws Exception { + return createDAG("SixLevelsFailingDAG", conf); + } } diff --git a/tez-tests/src/test/java/org/apache/tez/test/dag/ThreeLevelsFailingDAG.java b/tez-tests/src/test/java/org/apache/tez/test/dag/ThreeLevelsFailingDAG.java index 7f2e4f8ecd..da3f95d7fd 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/dag/ThreeLevelsFailingDAG.java +++ b/tez-tests/src/test/java/org/apache/tez/test/dag/ThreeLevelsFailingDAG.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,6 +17,7 @@ */ package org.apache.tez.test.dag; + import org.apache.hadoop.conf.Configuration; import org.apache.tez.common.TezUtils; import org.apache.tez.dag.api.DAG; @@ -28,11 +29,11 @@ * A DAG with vertices divided into 3 levels. * Vertex name is "lv". Level/vertex numbers start at 1. * Each vertex has failing processor and failing inputs. The constructor can accept Tez Configuration to indicate failing patterns. - * + * * DAG is shown with a diagram below. * Each vertex has its degree of parallelism indicated in a bracket following its name. * Each edge annotates with data movement (s = scatter/gather, b = broadcast) - * + * * l1v1(1) l1v2(2) l1v3(3) l1v4(2) * |s |s |s |b * | | | | @@ -43,33 +44,33 @@ * */ public class ThreeLevelsFailingDAG extends TwoLevelsFailingDAG { - - protected static Vertex l3v1, l3v2; - - protected static void addDAGVerticesAndEdges() { - TwoLevelsFailingDAG.addDAGVerticesAndEdges(); - l3v1 = Vertex.create("l3v1", TestProcessor.getProcDesc(payload), 4, defaultResource); - dag.addVertex(l3v1); - addEdge(l2v1, l3v1, DataMovementType.SCATTER_GATHER); - addEdge(l2v2, l3v1, DataMovementType.SCATTER_GATHER); - l3v2 = Vertex.create("l3v2", TestProcessor.getProcDesc(payload), 4, defaultResource); - dag.addVertex(l3v2); - addEdge(l2v2, l3v2, DataMovementType.BROADCAST); - addEdge(l2v3, l3v2, DataMovementType.SCATTER_GATHER); - addEdge(l2v4, l3v2, DataMovementType.SCATTER_GATHER); - } - - public static DAG createDAG(String name, - Configuration conf) throws Exception { - if (conf != null) { - payload = TezUtils.createUserPayloadFromConf(conf); - } - dag = DAG.create(name); - addDAGVerticesAndEdges(); - return dag; - } - - public static DAG createDAG(Configuration conf) throws Exception { - return createDAG("ThreeLevelsFailingDAG", conf); + + protected static Vertex l3v1, l3v2; + + protected static void addDAGVerticesAndEdges() { + TwoLevelsFailingDAG.addDAGVerticesAndEdges(); + l3v1 = Vertex.create("l3v1", TestProcessor.getProcDesc(payload), 4, defaultResource); + dag.addVertex(l3v1); + addEdge(l2v1, l3v1, DataMovementType.SCATTER_GATHER); + addEdge(l2v2, l3v1, DataMovementType.SCATTER_GATHER); + l3v2 = Vertex.create("l3v2", TestProcessor.getProcDesc(payload), 4, defaultResource); + dag.addVertex(l3v2); + addEdge(l2v2, l3v2, DataMovementType.BROADCAST); + addEdge(l2v3, l3v2, DataMovementType.SCATTER_GATHER); + addEdge(l2v4, l3v2, DataMovementType.SCATTER_GATHER); + } + + public static DAG createDAG(String name, + Configuration conf) throws Exception { + if (conf != null) { + payload = TezUtils.createUserPayloadFromConf(conf); } + dag = DAG.create(name); + addDAGVerticesAndEdges(); + return dag; + } + + public static DAG createDAG(Configuration conf) throws Exception { + return createDAG("ThreeLevelsFailingDAG", conf); + } } diff --git a/tez-tests/src/test/java/org/apache/tez/test/dag/TwoLevelsFailingDAG.java b/tez-tests/src/test/java/org/apache/tez/test/dag/TwoLevelsFailingDAG.java index 151e3855e1..63611f9f5b 100644 --- a/tez-tests/src/test/java/org/apache/tez/test/dag/TwoLevelsFailingDAG.java +++ b/tez-tests/src/test/java/org/apache/tez/test/dag/TwoLevelsFailingDAG.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,6 +17,7 @@ */ package org.apache.tez.test.dag; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.tez.common.TezUtils; @@ -36,11 +37,11 @@ * A DAG with vertices divided into 2 levels. * Vertex name is "lv". Level/vertex numbers start at 1. * Each vertex has failing processor and failing inputs. The constructor can accept Tez Configuration to indicate failing patterns. - * + * * DAG is shown with a diagram below. * Each vertex has its degree of parallelism indicated in a bracket following its name. * Each edge annotates with data movement (s = scatter/gather, b = broadcast) - * + * * l1v1(1) l1v2(2) l1v3(3) l1v4(2) * |s |s |s |b * | | | | @@ -48,66 +49,66 @@ * */ public class TwoLevelsFailingDAG { - static Resource defaultResource = Resource.newInstance(100, 0); - protected static DAG dag; - protected static UserPayload payload = UserPayload.create(null); - protected static Vertex l1v1, l1v2, l1v3, l1v4; - protected static Vertex l2v1, l2v2, l2v3, l2v4; + static Resource defaultResource = Resource.newInstance(100, 0); + protected static DAG dag; + protected static UserPayload payload = UserPayload.create(null); + protected static Vertex l1v1, l1v2, l1v3, l1v4; + protected static Vertex l2v1, l2v2, l2v3, l2v4; - public static DAG createDAG(String name, - Configuration conf) throws Exception { - if (conf != null) { - payload = TezUtils.createUserPayloadFromConf(conf); - } - dag = DAG.create(name); - addDAGVerticesAndEdges(); - return dag; - } - - protected static void addDAGVerticesAndEdges() { - l1v1 = Vertex.create("l1v1", TestProcessor.getProcDesc(payload), 1, defaultResource); - l2v1 = Vertex.create("l2v1", TestProcessor.getProcDesc(payload), 1, defaultResource); - addVerticesAndEdgeInternal(l1v1, l2v1, DataMovementType.SCATTER_GATHER); - l1v2 = Vertex.create("l1v2", TestProcessor.getProcDesc(payload), 2, defaultResource); - l2v2 = Vertex.create("l2v2", TestProcessor.getProcDesc(payload), 3, defaultResource); - addVerticesAndEdgeInternal(l1v2, l2v2, DataMovementType.SCATTER_GATHER); - l1v3 = Vertex.create("l1v3", TestProcessor.getProcDesc(payload), 3, defaultResource); - l2v3 = Vertex.create("l2v3", TestProcessor.getProcDesc(payload), 2, defaultResource); - addVerticesAndEdgeInternal(l1v3, l2v3, DataMovementType.SCATTER_GATHER); - l1v4 = Vertex.create("l1v4", TestProcessor.getProcDesc(payload), 2, defaultResource); - l2v4 = Vertex.create("l2v4", TestProcessor.getProcDesc(payload), 3, defaultResource); - addVerticesAndEdgeInternal(l1v4, l2v4, DataMovementType.BROADCAST); - } - - /** - * Adds 2 vertices and an edge connecting them. - * Given two vertices must not exist. - * - * @param v1 vertice 1 - * @param v2 vertice 2 - * @param dataMovementType Data movement type - */ - protected static void addVerticesAndEdgeInternal(Vertex v1, Vertex v2, DataMovementType dataMovementType) { - dag.addVertex(v1).addVertex(v2); - addEdge(v1, v2, dataMovementType); - } - - /** - * Adds an edge to given 2 vertices. - * @param v1 vertice 1 - * @param v2 vertice 2 - * @param dataMovementType Data movement type - */ - protected static void addEdge(Vertex v1, Vertex v2, DataMovementType dataMovementType) { - dag.addEdge(Edge.create(v1, v2, - EdgeProperty.create(dataMovementType, - DataSourceType.PERSISTED, - SchedulingType.SEQUENTIAL, - TestOutput.getOutputDesc(payload), - TestInput.getInputDesc(payload)))); - } - - public static DAG createDAG(Configuration conf) throws Exception { - return createDAG("TwoLevelsFailingDAG", conf); + public static DAG createDAG(String name, + Configuration conf) throws Exception { + if (conf != null) { + payload = TezUtils.createUserPayloadFromConf(conf); } + dag = DAG.create(name); + addDAGVerticesAndEdges(); + return dag; + } + + protected static void addDAGVerticesAndEdges() { + l1v1 = Vertex.create("l1v1", TestProcessor.getProcDesc(payload), 1, defaultResource); + l2v1 = Vertex.create("l2v1", TestProcessor.getProcDesc(payload), 1, defaultResource); + addVerticesAndEdgeInternal(l1v1, l2v1, DataMovementType.SCATTER_GATHER); + l1v2 = Vertex.create("l1v2", TestProcessor.getProcDesc(payload), 2, defaultResource); + l2v2 = Vertex.create("l2v2", TestProcessor.getProcDesc(payload), 3, defaultResource); + addVerticesAndEdgeInternal(l1v2, l2v2, DataMovementType.SCATTER_GATHER); + l1v3 = Vertex.create("l1v3", TestProcessor.getProcDesc(payload), 3, defaultResource); + l2v3 = Vertex.create("l2v3", TestProcessor.getProcDesc(payload), 2, defaultResource); + addVerticesAndEdgeInternal(l1v3, l2v3, DataMovementType.SCATTER_GATHER); + l1v4 = Vertex.create("l1v4", TestProcessor.getProcDesc(payload), 2, defaultResource); + l2v4 = Vertex.create("l2v4", TestProcessor.getProcDesc(payload), 3, defaultResource); + addVerticesAndEdgeInternal(l1v4, l2v4, DataMovementType.BROADCAST); + } + + /** + * Adds 2 vertices and an edge connecting them. + * Given two vertices must not exist. + * + * @param v1 vertice 1 + * @param v2 vertice 2 + * @param dataMovementType Data movement type + */ + protected static void addVerticesAndEdgeInternal(Vertex v1, Vertex v2, DataMovementType dataMovementType) { + dag.addVertex(v1).addVertex(v2); + addEdge(v1, v2, dataMovementType); + } + + /** + * Adds an edge to given 2 vertices. + * @param v1 vertice 1 + * @param v2 vertice 2 + * @param dataMovementType Data movement type + */ + protected static void addEdge(Vertex v1, Vertex v2, DataMovementType dataMovementType) { + dag.addEdge(Edge.create(v1, v2, + EdgeProperty.create(dataMovementType, + DataSourceType.PERSISTED, + SchedulingType.SEQUENTIAL, + TestOutput.getOutputDesc(payload), + TestInput.getInputDesc(payload)))); + } + + public static DAG createDAG(Configuration conf) throws Exception { + return createDAG("TwoLevelsFailingDAG", conf); + } } diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/Analyzer.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/Analyzer.java index 1f0a7ad62c..f8d4757b45 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/Analyzer.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/Analyzer.java @@ -21,7 +21,6 @@ import org.apache.tez.dag.api.TezException; import org.apache.tez.history.parser.datamodel.DagInfo; - public interface Analyzer { /** diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/CSVResult.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/CSVResult.java index 1144f7643c..4972bf5e56 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/CSVResult.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/CSVResult.java @@ -68,7 +68,7 @@ public Iterator getRecordsIterator() { return Iterators.unmodifiableIterator(recordsList.iterator()); } - @SuppressWarnings({ "rawtypes", "unchecked" }) + @SuppressWarnings({"rawtypes", "unchecked"}) public void sort(Comparator comparator) { Collections.sort(recordsList, comparator); } @@ -77,15 +77,18 @@ public void setComments(String comments) { this.comments = comments; } - @Override public String toJson() throws TezException { + @Override + public String toJson() throws TezException { return ""; } - @Override public String getComments() { + @Override + public String getComments() { return comments; } - @Override public String toString() { + @Override + public String toString() { return "CSVResult{" + "headers=" + Arrays.toString(headers) + ", recordsList=" + recordsList + @@ -107,7 +110,7 @@ public void dumpToFile(String fileName) throws IOException { } StringBuilder sb = new StringBuilder(); - for(int i=0;i + * http://www.apache.org/licenses/LICENSE-2.0 + *

    * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,7 +22,7 @@ public class AnalyzerDriver { - public static void main(String argv[]){ + public static void main(String argv[]) { int exitCode = -1; ProgramDriver pgd = new ProgramDriver(); try { @@ -61,11 +61,10 @@ public static void main(String argv[]){ pgd.addClass("TaskHangAnalyzer", HungTaskAnalyzer.class, "Print all vertices/tasks and their last attempts with status/duration/node"); exitCode = pgd.run(argv); - } catch(Throwable e){ + } catch (Throwable e) { e.printStackTrace(); } System.exit(exitCode); } - -} \ No newline at end of file +} diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/ContainerReuseAnalyzer.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/ContainerReuseAnalyzer.java index 553ff0e2cc..d22a7d1394 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/ContainerReuseAnalyzer.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/ContainerReuseAnalyzer.java @@ -33,14 +33,13 @@ import java.util.List; - /** * Get container reuse information at a per vertex level basis. */ public class ContainerReuseAnalyzer extends TezAnalyzerBase implements Analyzer { private static final String[] headers = - { "vertexName", "taskAttempts", "node", "containerId", "reuseCount" }; + {"vertexName", "taskAttempts", "node", "containerId", "reuseCount"}; private final CSVResult csvResult; diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/CriticalPathAnalyzer.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/CriticalPathAnalyzer.java index 3f5e3004b8..7ec2ed9e16 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/CriticalPathAnalyzer.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/CriticalPathAnalyzer.java @@ -80,33 +80,39 @@ public enum EntityType { long startCriticalPathTime; // time at which attempt is on critical path long stopCriticalPathTime; // time at which attempt is off critical path List notes = Lists.newLinkedList(); - + public CriticalPathStep(TaskAttemptInfo attempt, EntityType type) { this.type = type; this.attempt = attempt; } + public EntityType getType() { return type; } + public TaskAttemptInfo getAttempt() { return attempt; } + public long getStartCriticalTime() { return startCriticalPathTime; } + public long getStopCriticalTime() { return stopCriticalPathTime; } + public CriticalPathDependency getReason() { return reason; } + public List getNotes() { return notes; } } - + List criticalPath = Lists.newLinkedList(); - + Map attempts = Maps.newHashMap(); int maxConcurrency = 0; @@ -120,7 +126,7 @@ public CriticalPathAnalyzer(Configuration conf) { super(conf); } - @Override + @Override public void analyze(DagInfo dagInfo) throws TezException { // get all attempts in the dag and find the last failed/succeeded attempt. // ignore killed attempt to handle kills that happen upon dag completion @@ -128,7 +134,7 @@ public void analyze(DagInfo dagInfo) throws TezException { long lastAttemptFinishTime = 0; for (VertexInfo vertex : dagInfo.getVertices()) { for (TaskInfo task : vertex.getTasks()) { - for (TaskAttemptInfo attempt : task.getTaskAttempts()) { + for (TaskAttemptInfo attempt : task.getTaskAttempts()) { attempts.put(attempt.getTaskAttemptId(), attempt); if (attempt.getStatus().equals(SUCCEEDED_STATE) || attempt.getStatus().equals(FAILED_STATE)) { @@ -140,25 +146,25 @@ public void analyze(DagInfo dagInfo) throws TezException { } } } - + if (lastAttempt == null) { LOG.info("Cannot find last attempt to finish in DAG " + dagInfo.getDagId()); return; } - + createCriticalPath(dagInfo, lastAttempt, lastAttemptFinishTime, attempts); - + analyzeCriticalPath(dagInfo); if (getConf().getBoolean(DRAW_SVG, true)) { saveCriticalPathAsSVG(dagInfo); } } - + public List getCriticalPath() { return criticalPath; } - + private void saveCriticalPathAsSVG(DagInfo dagInfo) { SVGUtils svg = new SVGUtils(); String outputDir = getOutputDir(); @@ -169,44 +175,44 @@ private void saveCriticalPathAsSVG(DagInfo dagInfo) { LOG.info("Writing output to: " + outputFileName); svg.saveCriticalPathAsSVG(dagInfo, outputFileName, criticalPath); } - + static class TimeInfo implements Comparable { long timestamp; int count; boolean start; + TimeInfo(long timestamp, boolean start) { this.timestamp = timestamp; this.start = start; } - + @Override public int compareTo(TimeInfo o) { return Long.compare(this.timestamp, o.timestamp); } - + @Override public int hashCode() { - return (int)((timestamp >> 32) ^ timestamp); + return (int) ((timestamp >> 32) ^ timestamp); } - + @Override public boolean equals(Object o) { if (this == o) { return true; } - if(o == null) { + if (o == null) { return false; } if (o.getClass() == this.getClass()) { TimeInfo other = (TimeInfo) o; return (this.compareTo(other) == 0); - } - else { + } else { return false; } } } - + private void determineConcurrency(DagInfo dag) { ArrayList timeInfo = Lists.newArrayList(); for (VertexInfo v : dag.getVertices()) { @@ -220,7 +226,7 @@ private void determineConcurrency(DagInfo dag) { } } Collections.sort(timeInfo); - + int concurrency = 0; TimeInfo lastTimeInfo = null; for (TimeInfo t : timeInfo) { @@ -229,7 +235,7 @@ private void determineConcurrency(DagInfo dag) { if (lastTimeInfo == null || lastTimeInfo.timestamp < t.timestamp) { lastTimeInfo = t; lastTimeInfo.count = concurrency; - concurrencyByTime.add(lastTimeInfo); + concurrencyByTime.add(lastTimeInfo); } else { // lastTimeInfo.timestamp == t.timestamp lastTimeInfo.count = concurrency; @@ -239,7 +245,7 @@ private void determineConcurrency(DagInfo dag) { // System.out.println(t.timestamp + " " + t.count); // } } - + private int getIntervalMaxConcurrency(long begin, long end) { int concurrency = 0; for (TimeInfo timeInfo : concurrencyByTime) { @@ -255,7 +261,7 @@ private int getIntervalMaxConcurrency(long begin, long end) { } return concurrency; } - + private void analyzeAllocationOverhead(DagInfo dag) { List preemptedAttempts = Lists.newArrayList(); for (VertexInfo v : dag.getVertices()) { @@ -275,7 +281,7 @@ private void analyzeAllocationOverhead(DagInfo dag) { if (step.getType() != EntityType.ATTEMPT) { continue; } - + long creationTime = attempt.getCreationTime(); long allocationTime = attempt.getAllocationTime(); long finishTime = attempt.getFinishTime(); @@ -311,14 +317,14 @@ private void analyzeAllocationOverhead(DagInfo dag) { // for containerAttempts that used the container while this attempt was waiting // add up time container was allocated to containerAttempt. Account for allocations // that started before this attempt was created. - containerPreviousAllocatedTime += + containerPreviousAllocatedTime += (cFinishTime - (cAllocTime > creationTime ? cAllocTime : creationTime)); } } int numVertexTasks = attempt.getTaskInfo().getVertexInfo().getNumTasks(); int intervalMaxConcurrency = getIntervalMaxConcurrency(creationTime, finishTime); double numWaves = getWaves(numVertexTasks, intervalMaxConcurrency); - + if (reUsesForVertex > 1) { step.notes.add("Container ran multiple tasks for this vertex. "); if (numWaves < 1) { @@ -332,9 +338,9 @@ private void analyzeAllocationOverhead(DagInfo dag) { if (containerPreviousAllocatedTime >= attempt.getCreationToAllocationTimeInterval()) { step.notes.add("Container was fully allocated"); } else { - step.notes.add("Container in use for " + - SVGUtils.getTimeStr(containerPreviousAllocatedTime) + " out of " + - SVGUtils.getTimeStr(attempt.getCreationToAllocationTimeInterval()) + + step.notes.add("Container in use for " + + SVGUtils.getTimeStr(containerPreviousAllocatedTime) + " out of " + + SVGUtils.getTimeStr(attempt.getCreationToAllocationTimeInterval()) + " of allocation wait time"); } } @@ -354,13 +360,13 @@ private void analyzeAllocationOverhead(DagInfo dag) { } } } - + private double getWaves(int numTasks, int concurrency) { - double numWaves = (numTasks*1.0) / concurrency; - numWaves = (double)Math.round(numWaves * 10d) / 10d; // convert to 1 decimal place + double numWaves = (numTasks * 1.0) / concurrency; + numWaves = (double) Math.round(numWaves * 10d) / 10d; // convert to 1 decimal place return numWaves; } - + private void analyzeWaves(DagInfo dag) { for (int i = 0; i < criticalPath.size(); ++i) { CriticalPathStep step = criticalPath.get(i); @@ -382,14 +388,14 @@ private void analyzeWaves(DagInfo dag) { + " tasks in " + numWaves + " waves with available concurrency of " + intervalMaxConcurrency); if (numWaves > 1) { - if (numWaves%1 < 0.5) { + if (numWaves % 1 < 0.5) { // more than 1 wave needed and last wave is small step.notes.add("Last partial wave did not use full concurrency. "); } } } } - + private void analyzeStragglers(DagInfo dag) { long dagStartTime = dag.getStartTime(); long dagTime = dag.getFinishTime() - dagStartTime; @@ -412,17 +418,17 @@ private void analyzeStragglers(DagInfo dag) { long attemptExecTime = attempt.getPostDataExecutionTimeInterval(); if (avgPostDataExecutionTime * 1.25 < attemptExecTime) { step.notes - .add("Potential straggler. Post Data Execution time " + + .add("Potential straggler. Post Data Execution time " + SVGUtils.getTimeStr(attemptExecTime) - + " compared to vertex average of " + + + " compared to vertex average of " + SVGUtils.getTimeStr(avgPostDataExecutionTime)); } } } LOG.debug("DAG time taken: " + dagTime + " TotalAttemptTime: " + totalAttemptCriticalTime - + " DAG finish time: " + dag.getFinishTime() + " DAG start time: " + dagStartTime); + + " DAG finish time: " + dag.getFinishTime() + " DAG start time: " + dagStartTime); } - + private void analyzeCriticalPath(DagInfo dag) { if (!criticalPath.isEmpty()) { determineConcurrency(dag); @@ -431,9 +437,9 @@ private void analyzeCriticalPath(DagInfo dag) { analyzeAllocationOverhead(dag); } } - + private void createCriticalPath(DagInfo dagInfo, TaskAttemptInfo lastAttempt, - long lastAttemptFinishTime, Map attempts) { + long lastAttemptFinishTime, Map attempts) { List tempCP = Lists.newLinkedList(); if (lastAttempt != null) { TaskAttemptInfo currentAttempt = lastAttempt; @@ -455,7 +461,7 @@ private void createCriticalPath(DagInfo dagInfo, TaskAttemptInfo lastAttempt, Preconditions.checkState(currentAttempt != null); Preconditions.checkState(currentAttemptStopCriticalPathTime > 0); LOG.debug("Step: " + tempCP.size() + " Attempt: " + currentAttempt.getTaskAttemptId()); - + currentStep = new CriticalPathStep(currentAttempt, EntityType.ATTEMPT); currentStep.stopCriticalPathTime = currentAttemptStopCriticalPathTime; @@ -464,7 +470,7 @@ private void createCriticalPath(DagInfo dagInfo, TaskAttemptInfo lastAttempt, long currentStepLastDataEventTime = 0; String currentStepLastDataTA = null; DataDependencyEvent item = currentAttempt.getLastDataEventInfo(currentStep.stopCriticalPathTime); - if (item!=null) { + if (item != null) { currentStepLastDataEventTime = item.getTimestamp(); currentStepLastDataTA = item.getTaskAttemptId(); } @@ -477,25 +483,25 @@ private void createCriticalPath(DagInfo dagInfo, TaskAttemptInfo lastAttempt, // this should only happen for read errors in currentAttempt List dataEvents = currentAttempt.getLastDataEvents(); Preconditions.checkState(dataEvents.size() > 1); // received - // original and - // retry data events + // original and + // retry data events Preconditions.checkState(currentStepLastDataEventTime < dataEvents .get(dataEvents.size() - 1).getTimestamp()); // new event is - // earlier than - // last + // earlier than + // last } } } tempCP.add(currentStep); - + // find the next attempt on the critical path boolean dataDependency = false; // find out predecessor dependency if (currentStepLastDataEventTime > currentAttempt.getCreationTime()) { dataDependency = true; } - + long startCriticalPathTime = 0; String nextAttemptId = null; CriticalPathDependency reason = null; @@ -532,7 +538,7 @@ private void createCriticalPath(DagInfo dagInfo, TaskAttemptInfo lastAttempt, if (nextAttemptId != null) { VertexInfo currentVertex = currentAttempt.getTaskInfo().getVertexInfo(); VertexInfo nextVertex = nextAttempt.getTaskInfo().getVertexInfo(); - if (!nextVertex.getVertexName().equals(currentVertex.getVertexName())){ + if (!nextVertex.getVertexName().equals(currentVertex.getVertexName())) { // cause from different vertex. Might be rerun to re-generate outputs for (VertexInfo outVertex : currentVertex.getOutputVertices()) { if (nextVertex.getVertexName().equals(outVertex.getVertexName())) { @@ -582,9 +588,9 @@ private void createCriticalPath(DagInfo dagInfo, TaskAttemptInfo lastAttempt, currentStep.startCriticalPathTime = startCriticalPathTime; currentStep.reason = reason; - + Preconditions.checkState(currentStep.stopCriticalPathTime >= currentStep.startCriticalPathTime); - + if (Strings.isNullOrEmpty(nextAttemptId)) { Preconditions.checkState(reason.equals(CriticalPathDependency.INIT_DEPENDENCY)); Preconditions.checkState(startCriticalPathTime == 0); @@ -592,7 +598,7 @@ private void createCriticalPath(DagInfo dagInfo, TaskAttemptInfo lastAttempt, // assume attempts start critical path time is when its scheduled. before that is // vertex initialization time currentStep.startCriticalPathTime = currentStep.attempt.getCreationTime(); - + // add vertex init step long initStepStopCriticalTime = currentStep.startCriticalPathTime; currentStep = new CriticalPathStep(currentAttempt, EntityType.VERTEX_INIT); @@ -600,33 +606,33 @@ private void createCriticalPath(DagInfo dagInfo, TaskAttemptInfo lastAttempt, currentStep.startCriticalPathTime = dagInfo.getStartTime(); currentStep.reason = CriticalPathDependency.INIT_DEPENDENCY; tempCP.add(currentStep); - + if (!tempCP.isEmpty()) { - for (int i=tempCP.size() - 1; i>=0; --i) { + for (int i = tempCP.size() - 1; i >= 0; --i) { criticalPath.add(tempCP.get(i)); } } return; } - + currentAttempt = attempts.get(nextAttemptId); currentAttemptStopCriticalPathTime = startCriticalPathTime; } } } - + @Override public CSVResult getResult() throws TezException { - String[] headers = { "Entity", "PathReason", "Status", "CriticalStartTime", - "CriticalStopTime", "Notes" }; + String[] headers = {"Entity", "PathReason", "Status", "CriticalStartTime", + "CriticalStopTime", "Notes"}; CSVResult csvResult = new CSVResult(headers); for (CriticalPathStep step : criticalPath) { String entity = (step.getType() == EntityType.ATTEMPT ? step.getAttempt().getTaskAttemptId() : (step.getType() == EntityType.VERTEX_INIT - ? step.attempt.getTaskInfo().getVertexInfo().getVertexName() : "DAG COMMIT")); - String [] record = {entity, step.getReason().name(), - step.getAttempt().getDetailedStatus(), String.valueOf(step.getStartCriticalTime()), + ? step.attempt.getTaskInfo().getVertexInfo().getVertexName() : "DAG COMMIT")); + String[] record = {entity, step.getReason().name(), + step.getAttempt().getDetailedStatus(), String.valueOf(step.getStartCriticalTime()), String.valueOf(step.getStopCriticalTime()), Joiner.on(";").join(step.getNotes())}; csvResult.addRecord(record); @@ -649,5 +655,4 @@ public static void main(String[] args) throws Exception { int res = ToolRunner.run(config, new CriticalPathAnalyzer(config), args); System.exit(res); } - } diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/DagOverviewAnalyzer.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/DagOverviewAnalyzer.java index b193c30a90..7ce5e7f722 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/DagOverviewAnalyzer.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/DagOverviewAnalyzer.java @@ -35,7 +35,7 @@ public class DagOverviewAnalyzer extends TezAnalyzerBase implements Analyzer { private final String[] headers = - { "name", "id", "event_type", "status", "event_time", "event_time_str", "vertex_task_stats", "diagnostics" }; + {"name", "id", "event_type", "status", "event_time", "event_time_str", "vertex_task_stats", "diagnostics"}; private final CSVResult csvResult; private static final SimpleDateFormat FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); @@ -47,8 +47,8 @@ public DagOverviewAnalyzer(Configuration config) { @Override public void analyze(DagInfo dagInfo) throws TezException { for (Event event : dagInfo.getEvents()) { - csvResult.addRecord(new String[] { dagInfo.getDagId(), dagInfo.getDagId(), event.getType(), - dagInfo.getStatus(), Long.toString(event.getTime()), toDateStr(event.getTime()), "", "" }); + csvResult.addRecord(new String[]{dagInfo.getDagId(), dagInfo.getDagId(), event.getType(), + dagInfo.getStatus(), Long.toString(event.getTime()), toDateStr(event.getTime()), "", ""}); } for (VertexInfo vertex : dagInfo.getVertices()) { for (Event event : vertex.getEvents()) { @@ -60,30 +60,30 @@ public void analyze(DagInfo dagInfo) throws TezException { break; } } - csvResult.addRecord(new String[] { vertex.getVertexName(), vertex.getVertexId(), + csvResult.addRecord(new String[]{vertex.getVertexName(), vertex.getVertexId(), event.getType(), vertex.getStatus(), Long.toString(event.getTime()), - toDateStr(event.getTime()), getTaskStats(vertex), vertexFailureInfoIfAny }); + toDateStr(event.getTime()), getTaskStats(vertex), vertexFailureInfoIfAny}); } // a failed task can lead to dag failure, so hopefully holds valuable information for (TaskInfo failedTask : vertex.getFailedTasks()) { for (Event failedTaskEvent : failedTask.getEvents()) { if (failedTaskEvent.getType().equalsIgnoreCase("TASK_FINISHED")) { - csvResult.addRecord(new String[] { vertex.getVertexName(), failedTask.getTaskId(), + csvResult.addRecord(new String[]{vertex.getVertexName(), failedTask.getTaskId(), failedTaskEvent.getType(), failedTask.getStatus(), Long.toString(failedTaskEvent.getTime()), toDateStr(failedTaskEvent.getTime()), getTaskStats(vertex), - failedTask.getDiagnostics().replaceAll(",", " ").replaceAll("\n", " ") }); + failedTask.getDiagnostics().replaceAll(",", " ").replaceAll("\n", " ")}); } } // if we already found a failing task, let's scan the failing attempts as well for (TaskAttemptInfo failedAttempt : failedTask.getFailedTaskAttempts()) { for (Event failedTaskAttemptEvent : failedAttempt.getEvents()) { if (failedTaskAttemptEvent.getType().equalsIgnoreCase("TASK_ATTEMPT_FINISHED")) { - csvResult.addRecord(new String[] { vertex.getVertexName(), + csvResult.addRecord(new String[]{vertex.getVertexName(), failedAttempt.getTaskAttemptId(), failedTaskAttemptEvent.getType(), failedAttempt.getStatus(), Long.toString(failedTaskAttemptEvent.getTime()), toDateStr(failedTaskAttemptEvent.getTime()), getTaskStats(vertex), - failedAttempt.getDiagnostics().replaceAll(",", " ").replaceAll("\n", " ") }); + failedAttempt.getDiagnostics().replaceAll(",", " ").replaceAll("\n", " ")}); } } } diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/HungTaskAnalyzer.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/HungTaskAnalyzer.java index 9a38e28ba2..c35b88ba64 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/HungTaskAnalyzer.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/HungTaskAnalyzer.java @@ -37,8 +37,8 @@ * Gives insights about hanging task attempts by providing details about last attempts of all tasks. */ public class HungTaskAnalyzer extends TezAnalyzerBase implements Analyzer { - private final String[] headers = { "vertex", "task", " number_of_attempts", "last_attempt_id", - "last_attempt_status", "last_attempt_duration_ms", "last_attempt_node" }; + private final String[] headers = {"vertex", "task", " number_of_attempts", "last_attempt_id", + "last_attempt_status", "last_attempt_duration_ms", "last_attempt_node"}; private final CSVResult csvResult; private static final String HEADER_NUM_ATTEMPTS = "num_attempts"; @@ -77,7 +77,7 @@ public void analyze(DagInfo dagInfo) throws TezException { thisTaskData.put(HEADER_LAST_ATTEMPT_DURATION_MS, (attempt.getFinishTime() == 0 || attempt.getStartTime() == 0) ? "-1" - : Long.toString(attempt.getFinishTime() - attempt.getStartTime())); + : Long.toString(attempt.getFinishTime() - attempt.getStartTime())); } } for (Map.Entry> task : taskData.entrySet()) { @@ -96,15 +96,15 @@ public int compare(String[] first, String[] second) { int attemptNumberOrder = Integer.valueOf(second[2]).compareTo(Integer.valueOf(first[2])); return vertexOrder == 0 - ? (lastAttemptStatusOrder == 0 ? attemptNumberOrder : lastAttemptStatusOrder) - : vertexOrder; + ? (lastAttemptStatusOrder == 0 ? attemptNumberOrder : lastAttemptStatusOrder) + : vertexOrder; } }); } private void addARecord(String vertexName, String taskId, String numAttempts, - String lastAttemptId, String lastAttemptStatus, String lastAttemptDuration, - String lastAttemptNode) { + String lastAttemptId, String lastAttemptStatus, String lastAttemptDuration, + String lastAttemptNode) { String[] record = new String[7]; record[0] = vertexName; record[1] = taskId; diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/InputReadErrorAnalyzer.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/InputReadErrorAnalyzer.java index 3cb523ff9a..125c228fbb 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/InputReadErrorAnalyzer.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/InputReadErrorAnalyzer.java @@ -34,7 +34,7 @@ * Helps finding the root cause of shuffle errors, e.g. which node(s) can be blamed for them. */ public class InputReadErrorAnalyzer extends TezAnalyzerBase implements Analyzer { - private final String[] headers = { "vertex:attempt", "status", "time", "node", "diagnostics" }; + private final String[] headers = {"vertex:attempt", "status", "time", "node", "diagnostics"}; private final CSVResult csvResult; public InputReadErrorAnalyzer(Configuration config) { @@ -52,10 +52,10 @@ public void analyze(DagInfo dagInfo) throws TezException { || "NODE_FAILED".equalsIgnoreCase(terminationCause)) { for (Event event : attempt.getEvents()) { if (event.getType().equalsIgnoreCase("TASK_ATTEMPT_FINISHED")) { - csvResult.addRecord(new String[] { + csvResult.addRecord(new String[]{ vertex.getVertexName() + ":" + attempt.getTaskAttemptId(), attempt.getDetailedStatus(), String.valueOf(event.getTime()), attempt.getNodeId(), - attempt.getDiagnostics().replaceAll(",", " ").replaceAll("\n", " ") }); + attempt.getDiagnostics().replaceAll(",", " ").replaceAll("\n", " ")}); } } } diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/LocalityAnalyzer.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/LocalityAnalyzer.java index d640704f93..df18fd8871 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/LocalityAnalyzer.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/LocalityAnalyzer.java @@ -36,7 +36,6 @@ import java.util.List; import java.util.Map; - /** * Get locality information for tasks for vertices and get their task execution times. * This would be helpeful to co-relate if the vertex runtime is anyways related to the data @@ -44,10 +43,10 @@ */ public class LocalityAnalyzer extends TezAnalyzerBase implements Analyzer { - private final String[] headers = { "vertexName", "numTasks", "dataLocalRatio", "rackLocalRatio", + private final String[] headers = {"vertexName", "numTasks", "dataLocalRatio", "rackLocalRatio", "otherRatio", "avgDataLocalTaskRuntime", "avgRackLocalTaskRuntime", "avgOtherLocalTaskRuntime", "noOfInputs", "avgHDFSBytesRead_DataLocal", - "avgHDFSBytesRead_RackLocal", "avgHDFSBytesRead_Others", "recommendation" }; + "avgHDFSBytesRead_RackLocal", "avgHDFSBytesRead_Others", "recommendation"}; private static final String DATA_LOCAL_RATIO = "tez.locality-analyzer.data.local.ratio"; private static final float DATA_LOCAL_RATIO_DEFAULT = 0.5f; @@ -144,7 +143,7 @@ private TaskAttemptDetails computeAverages(VertexInfo vertexInfo, DAGCounter cou TaskAttemptDetails result = new TaskAttemptDetails(); - for(TaskAttemptInfo attemptInfo : vertexInfo.getTaskAttempts()) { + for (TaskAttemptInfo attemptInfo : vertexInfo.getTaskAttempts()) { Map localityCounter = attemptInfo.getCounter(DAGCounter.class.getName(), counter.toString()); @@ -156,7 +155,7 @@ private TaskAttemptDetails computeAverages(VertexInfo vertexInfo, DAGCounter cou //get HDFSBytes read counter Map hdfsBytesReadCounter = attemptInfo.getCounter(FileSystemCounter .class.getName(), FileSystemCounter.HDFS_BYTES_READ.name()); - for(Map.Entry entry : hdfsBytesReadCounter.entrySet()) { + for (Map.Entry entry : hdfsBytesReadCounter.entrySet()) { totalHDFSBytesRead += entry.getValue().getValue(); } } @@ -168,15 +167,18 @@ private TaskAttemptDetails computeAverages(VertexInfo vertexInfo, DAGCounter cou return result; } - @Override public CSVResult getResult() throws TezException { + @Override + public CSVResult getResult() throws TezException { return csvResult; } - @Override public String getName() { + @Override + public String getName() { return "Locality Analyzer"; } - @Override public String getDescription() { + @Override + public String getDescription() { return "Analyze for locality information (data local, rack local, off-rack)"; } diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/OneOnOneEdgeAnalyzer.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/OneOnOneEdgeAnalyzer.java index a6cb3f1511..b5ddcdaae6 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/OneOnOneEdgeAnalyzer.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/OneOnOneEdgeAnalyzer.java @@ -50,8 +50,8 @@ public class OneOnOneEdgeAnalyzer extends TezAnalyzerBase implements Analyzer { private static final Logger LOG = LoggerFactory.getLogger(OneOnOneEdgeAnalyzer.class); - private final String[] headers = { "sourceVertex", "downstreamVertex", "srcTaskId", - "srcContainerHost", "destContainerHost" }; + private final String[] headers = {"sourceVertex", "downstreamVertex", "srcTaskId", + "srcContainerHost", "destContainerHost"}; // DataMovementType::ONE_TO_ONE private static final String ONE_TO_ONE = "ONE_TO_ONE"; @@ -87,7 +87,7 @@ public void analyze(DagInfo dagInfo) throws TezException { String downstreamContainerHost = downStreamTaskToContainerMap.get(taskId); if (downstreamContainerHost != null) { if (!sourceContainerHost.equalsIgnoreCase(downstreamContainerHost)) { - // downstream task got scheduled on different machine than src + // downstream task got scheduled on different machine than src LOG.info("TaskID: {}, source: {}, downStream:{}", taskId, sourceContainerHost, downstreamContainerHost); result.add(sourceVertex.getVertexName()); @@ -100,7 +100,7 @@ public void analyze(DagInfo dagInfo) throws TezException { missedCounter++; } } - result.clear(); + result.clear(); } LOG.info("Total tasks:{}, miss: {}", sourceTaskToContainerMap.size(), missedCounter); } @@ -145,4 +145,3 @@ public static void main(String[] args) throws Exception { System.exit(res); } } - diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/ShuffleTimeAnalyzer.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/ShuffleTimeAnalyzer.java index f8f9112bb7..54b3408918 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/ShuffleTimeAnalyzer.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/ShuffleTimeAnalyzer.java @@ -35,14 +35,12 @@ import java.util.List; import java.util.Map; - /** * Analyze the time taken by merge phase, shuffle phase, time taken to do realistic work etc in * tasks. - * + *

    * Just dump REDUCE_INPUT_GROUPS, REDUCE_INPUT_RECORDS, its ratio and SHUFFLE_BYTES for tasks * grouped by vertices. Provide time taken as well. Just render it as a table for now. - * */ public class ShuffleTimeAnalyzer extends TezAnalyzerBase implements Analyzer { @@ -58,18 +56,17 @@ public class ShuffleTimeAnalyzer extends TezAnalyzerBase implements Analyzer { private static final String MIN_SHUFFLE_RECORDS = "tez.shuffle-time-analyzer.shuffle.min.records"; private static final long MIN_SHUFFLE_RECORDS_DEFAULT = 10000; - private static final String[] headers = { "vertexName", "taskAttemptId", "Node", "counterGroup", + private static final String[] headers = {"vertexName", "taskAttemptId", "Node", "counterGroup", "Comments", "REDUCE_INPUT_GROUPS", "REDUCE_INPUT_RECORDS", "ratio", "SHUFFLE_BYTES", "TotalTime", "Time_taken_to_receive_all_events", "MERGE_PHASE_TIME", "SHUFFLE_PHASE_TIME", "TimeTaken_For_Real_Task", "FIRST_EVENT_RECEIVED", "LAST_EVENT_RECEIVED", - "SHUFFLE_BYTES_DISK_DIRECT" }; + "SHUFFLE_BYTES_DISK_DIRECT"}; private final CSVResult csvResult = new CSVResult(headers); private final float realWorkDoneRatio; private final long minShuffleRecords; - public ShuffleTimeAnalyzer(Configuration config) { super(config); @@ -97,7 +94,7 @@ public void analyze(DagInfo dagInfo) throws TezException { String counterGroupName = entry.getKey(); long reduceInputGroupsVal = entry.getValue().getValue(); long reduceInputRecordsVal = (reduceInputRecords.get(counterGroupName) != null) ? - reduceInputRecords.get(counterGroupName).getValue() : 0; + reduceInputRecords.get(counterGroupName).getValue() : 0; if (reduceInputRecordsVal <= 0) { continue; @@ -152,7 +149,6 @@ public void analyze(DagInfo dagInfo) throws TezException { } } } - } /** @@ -176,7 +172,7 @@ private String getOverheadFromSourceTasks(String counterGroupName, TaskAttemptIn } private String getCounterValue(TaskCounter counter, String counterGroupName, - TaskAttemptInfo attemptInfo) { + TaskAttemptInfo attemptInfo) { Map tezCounterMap = attemptInfo.getCounter(counter.toString()); if (tezCounterMap != null) { for (Map.Entry entry : tezCounterMap.entrySet()) { diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SkewAnalyzer.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SkewAnalyzer.java index b0387d1973..cddbf0faa2 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SkewAnalyzer.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SkewAnalyzer.java @@ -35,7 +35,6 @@ import java.util.Map; import java.util.Objects; - /** *

    * Identify the skew (RECORD_INPUT_GROUPS / REDUCE_INPUT_RECORDS) ratio for all task attempts @@ -77,11 +76,9 @@ public class SkewAnalyzer extends TezAnalyzerBase implements Analyzer { + ".group.max.ratio"; private static final float ATTEMPT_SHUFFLE_KEY_GROUP_MAX_RATIO_DEFAULT = 0.4f; - - - private static final String[] headers = { "vertexName", "taskAttemptId", "counterGroup", "node", + private static final String[] headers = {"vertexName", "taskAttemptId", "counterGroup", "node", "REDUCE_INPUT_GROUPS", "REDUCE_INPUT_RECORDS", "ratio", "SHUFFLE_BYTES", "timeTaken", - "observation" }; + "observation"}; private final CSVResult csvResult = new CSVResult(headers); @@ -128,7 +125,6 @@ private void analyzeGroupSkewPerSource(TaskAttemptInfo attemptInfo) { .REDUCE_INPUT_RECORDS.toString()); Map shuffleBytes = attemptInfo.getCounter(TaskCounter.SHUFFLE_BYTES.toString()); - //tez counter for every source for (Map.Entry entry : reduceInputGroups.entrySet()) { if (entry.getKey().equals(TaskCounter.class.getName())) { @@ -187,7 +183,6 @@ private void analyzeRecordSkewPerSource(TaskAttemptInfo attemptInfo) { .REDUCE_INPUT_RECORDS.toString()); Map shuffleBytes = attemptInfo.getCounter(TaskCounter.SHUFFLE_BYTES.toString()); - //tez counter for every source for (Map.Entry entry : reduceInputGroups.entrySet()) { if (entry.getKey().equals(TaskCounter.class.getName())) { @@ -200,7 +195,7 @@ private void analyzeRecordSkewPerSource(TaskAttemptInfo attemptInfo) { long inputGroupsCount = entry.getValue().getValue(); long inputRecordsCount = (reduceInputRecords.get(counterGroup) != null) ? reduceInputRecords .get(counterGroup).getValue() : 0; - long shuffleBytesPerSource = (shuffleBytes.get(counterGroup) != null) ?shuffleBytes.get + long shuffleBytesPerSource = (shuffleBytes.get(counterGroup) != null) ? shuffleBytes.get (counterGroup).getValue() : 0; long vertexLevelInputRecordsCount = (vertexLevelReduceInputRecords.get(counterGroup) != null) ? @@ -226,7 +221,6 @@ private void analyzeRecordSkewPerSource(TaskAttemptInfo attemptInfo) { + "Consider adjusting parallelism & check partition logic"); csvResult.addRecord(result.toArray(new String[result.size()])); - } } } @@ -285,11 +279,8 @@ private void analyzeForParallelism(TaskAttemptInfo attemptInfo) { } } } - - } - @Override public CSVResult getResult() throws TezException { return csvResult; diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowNodeAnalyzer.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowNodeAnalyzer.java index 9e573c2033..91e9ae31d1 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowNodeAnalyzer.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowNodeAnalyzer.java @@ -41,7 +41,6 @@ import java.util.Collection; import java.util.List; - /** * This will provide the set of nodes participated in the DAG in descending order of task execution * time. @@ -52,10 +51,10 @@ public class SlowNodeAnalyzer extends TezAnalyzerBase implements Analyzer { private static final Log LOG = LogFactory.getLog(SlowNodeAnalyzer.class); - private static final String[] headers = { "nodeName", "noOfTasksExecuted", "noOfKilledTasks", + private static final String[] headers = {"nodeName", "noOfTasksExecuted", "noOfKilledTasks", "noOfFailedTasks", "avgSucceededTaskExecutionTime", "avgKilledTaskExecutionTime", "avgFailedTaskExecutionTime", "avgHDFSBytesRead", "avgHDFSBytesWritten", - "avgFileBytesRead", "avgFileBytesWritten", "avgGCTimeMillis", "avgCPUTimeMillis" }; + "avgFileBytesRead", "avgFileBytesWritten", "avgGCTimeMillis", "avgCPUTimeMillis"}; private final CSVResult csvResult = new CSVResult(headers); @@ -99,17 +98,18 @@ public void analyze(DagInfo dagInfo) throws TezException { record.add(getAvgCounter(taskAttemptInfos, TaskCounter.class .getName(), TaskCounter.GC_TIME_MILLIS.name()) + ""); record.add(getAvgCounter(taskAttemptInfos, TaskCounter.class - .getName(), TaskCounter.CPU_MILLISECONDS.name()) + ""); + .getName(), TaskCounter.CPU_MILLISECONDS.name()) + ""); - csvResult.addRecord(record.toArray(new String[record.size()])); + csvResult.addRecord(record.toArray(new String[record.size()])); } } private Iterable getFilteredTaskAttempts(Collection - taskAttemptInfos, final TaskAttemptState status) { + taskAttemptInfos, final TaskAttemptState status) { return Iterables.filter(taskAttemptInfos, new Predicate() { - @Override public boolean apply(TaskAttemptInfo input) { + @Override + public boolean apply(TaskAttemptInfo input) { return input.getStatus().equalsIgnoreCase(status.toString()); } }); @@ -171,7 +171,7 @@ public String getDescription() { sb.append("This could be used to find out the set of nodes where the tasks are taking more " + "time on average.").append("\n"); sb.append("This could be used to find out the set of nodes where the tasks are taking more " - + "time on average and to understand whether too many tasks got scheduled on a node.") + + "time on average and to understand whether too many tasks got scheduled on a node.") .append("\n"); sb.append("One needs to combine the task execution time with other metrics like bytes " + "read/written etc to get better idea of bad nodes. In order to understand the slow " diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowTaskIdentifier.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowTaskIdentifier.java index 7c9958b250..27727bd4ba 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowTaskIdentifier.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowTaskIdentifier.java @@ -33,18 +33,17 @@ import java.util.Comparator; import java.util.List; - /** * Analyze slow tasks in the DAG. Top 100 tasks are listed by default. - * + *

    *

    * //TODO: We do not get counters for killed task attempts yet. */ public class SlowTaskIdentifier extends TezAnalyzerBase implements Analyzer { - private static final String[] headers = { "vertexName", "taskAttemptId", + private static final String[] headers = {"vertexName", "taskAttemptId", "Node", "taskDuration", "Status", "diagnostics", - "NoOfInputs" }; + "NoOfInputs"}; private final CSVResult csvResult; @@ -59,13 +58,14 @@ public SlowTaskIdentifier(Configuration config) { @Override public void analyze(DagInfo dagInfo) throws TezException { List taskAttempts = Lists.newArrayList(); - for(VertexInfo vertexInfo : dagInfo.getVertices()) { + for (VertexInfo vertexInfo : dagInfo.getVertices()) { taskAttempts.addAll(vertexInfo.getTaskAttempts()); } //sort them by runtime in descending order Collections.sort(taskAttempts, new Comparator() { - @Override public int compare(TaskAttemptInfo o1, TaskAttemptInfo o2) { + @Override + public int compare(TaskAttemptInfo o1, TaskAttemptInfo o2) { return (o1.getTimeTaken() > o2.getTimeTaken()) ? -1 : ((o1.getTimeTaken() == o2.getTimeTaken()) ? 0 : 1); @@ -91,7 +91,6 @@ public void analyze(DagInfo dagInfo) throws TezException { csvResult.addRecord(record.toArray(new String[record.size()])); } - } @Override diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowestVertexAnalyzer.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowestVertexAnalyzer.java index efa39a3223..977048fbd5 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowestVertexAnalyzer.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SlowestVertexAnalyzer.java @@ -42,10 +42,10 @@ */ public class SlowestVertexAnalyzer extends TezAnalyzerBase implements Analyzer { - private static final String[] headers = { "vertexName", "taskAttempts", "totalTime", + private static final String[] headers = {"vertexName", "taskAttempts", "totalTime", "shuffleTime", "shuffleTime_Max", "LastEventReceived", "LastEventReceivedFrom", "75thPercentile", "95thPercentile", "98thPercentile", "Median", - "observation", "comments" }; + "observation", "comments"}; private final CSVResult csvResult = new CSVResult(headers); @@ -61,7 +61,6 @@ public SlowestVertexAnalyzer(Configuration config) { super(config); this.vertexRuntimeThreshold = Math.max(1, config.getLong(MAX_VERTEX_RUNTIME, MAX_VERTEX_RUNTIME_DEFAULT)); - } private long getTaskRuntime(VertexInfo vertexInfo) { @@ -80,7 +79,7 @@ public void analyze(DagInfo dagInfo) throws TezException { for (VertexInfo vertexInfo : dagInfo.getVertices()) { String vertexName = vertexInfo.getVertexName(); - if (vertexInfo.getFirstTaskToStart() == null || vertexInfo.getLastTaskToFinish() == null) { + if (vertexInfo.getFirstTaskToStart() == null || vertexInfo.getLastTaskToFinish() == null) { continue; } @@ -90,7 +89,6 @@ public void analyze(DagInfo dagInfo) throws TezException { String maxSourceName = ""; taskAttemptRuntimeHistorgram = metrics.histogram(vertexName); - for (TaskAttemptInfo attemptInfo : vertexInfo.getTaskAttempts()) { taskAttemptRuntimeHistorgram.update(attemptInfo.getTimeTaken()); @@ -187,7 +185,6 @@ public void analyze(DagInfo dagInfo) throws TezException { } } - @Override public CSVResult getResult() throws TezException { return csvResult; diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SpillAnalyzerImpl.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SpillAnalyzerImpl.java index 026dd1593f..3447a85a36 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SpillAnalyzerImpl.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/SpillAnalyzerImpl.java @@ -35,7 +35,6 @@ import java.util.List; import java.util.Map; - /** * Find out tasks which have more than 1 spill (ADDITIONAL_SPILL_COUNT). *

    @@ -43,11 +42,11 @@ */ public class SpillAnalyzerImpl extends TezAnalyzerBase implements Analyzer { - private static final String[] headers = { "vertexName", "taskAttemptId", + private static final String[] headers = {"vertexName", "taskAttemptId", "Node", "counterGroupName", "spillCount", "taskDuration", "OUTPUT_BYTES", "OUTPUT_RECORDS", - "SPILLED_RECORDS", "Recommendation" }; + "SPILLED_RECORDS", "Recommendation"}; private final CSVResult csvResult; diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TaskAssignmentAnalyzer.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TaskAssignmentAnalyzer.java index 02b821f367..ad9e0eba8a 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TaskAssignmentAnalyzer.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TaskAssignmentAnalyzer.java @@ -36,7 +36,7 @@ */ public class TaskAssignmentAnalyzer extends TezAnalyzerBase implements Analyzer { - private final String[] headers = { "vertex", "node", "numTaskAttempts", "load" }; + private final String[] headers = {"vertex", "node", "numTaskAttempts", "load"}; private final CSVResult csvResult; public TaskAssignmentAnalyzer(Configuration config) { @@ -62,7 +62,7 @@ public void analyze(DagInfo dagInfo) throws TezException { } private void addARecord(String vertexName, String node, int numTasks, - double load) { + double load) { String[] record = new String[4]; record[0] = vertexName; record[1] = node; diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TaskAttemptResultStatisticsAnalyzer.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TaskAttemptResultStatisticsAnalyzer.java index cf6b2f0d8e..a4d0b6f299 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TaskAttemptResultStatisticsAnalyzer.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TaskAttemptResultStatisticsAnalyzer.java @@ -34,7 +34,7 @@ /** * Get simple count of task attempt states on vertex:node:status level, like below. - * + *

    * vertex (+task stats: all/succeeded/failed/killed),node,status,numAttempts * Map 1 (vertex_x_y_z) (216/153/0/63),node1,KILLED:INTERNAL_PREEMPTION,1185 * Map 1 (vertex_x_y_z) (216/153/0/63),node1,KILLED:TERMINATED_AT_SHUTDOWN,22 @@ -43,7 +43,7 @@ */ public class TaskAttemptResultStatisticsAnalyzer extends TezAnalyzerBase implements Analyzer { private final String[] headers = - { "vertex (+task stats: all/succeeded/failed/killed)", "node", "status", "numAttempts" }; + {"vertex (+task stats: all/succeeded/failed/killed)", "node", "status", "numAttempts"}; private final CSVResult csvResult; public TaskAttemptResultStatisticsAnalyzer(Configuration config) { @@ -86,7 +86,7 @@ public int compare(String[] first, String[] second) { } private void addARecord(String vertexData, String node, String status, - int numAttempts) { + int numAttempts) { String[] record = new String[4]; record[0] = vertexData; record[1] = node; diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TaskConcurrencyAnalyzer.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TaskConcurrencyAnalyzer.java index 91f51b4c21..3c4c0fc529 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TaskConcurrencyAnalyzer.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TaskConcurrencyAnalyzer.java @@ -38,7 +38,7 @@ */ public class TaskConcurrencyAnalyzer extends TezAnalyzerBase implements Analyzer { - private static final String[] headers = { "time", "vertexName", "concurrentTasksRunning" }; + private static final String[] headers = {"time", "vertexName", "concurrentTasksRunning"}; private final CSVResult csvResult; @@ -78,7 +78,8 @@ public void analyze(DagInfo dagInfo) throws TezException { * - Decrement concurrent tasks when start event is encountered */ TreeMultiset timeInfoSet = TreeMultiset.create(new Comparator() { - @Override public int compare(TimeInfo o1, TimeInfo o2) { + @Override + public int compare(TimeInfo o1, TimeInfo o2) { if (o1.timestamp < o2.timestamp) { return -1; } @@ -114,16 +115,16 @@ public void analyze(DagInfo dagInfo) throws TezException { //Compute concurrent tasks in the list now. int concurrentTasks = 0; - for(TimeInfo timeInfo : timeInfoSet.elementSet()) { + for (TimeInfo timeInfo : timeInfoSet.elementSet()) { switch (timeInfo.eventType) { - case START: - concurrentTasks += timeInfoSet.count(timeInfo); - break; - case FINISH: - concurrentTasks -= timeInfoSet.count(timeInfo); - break; - default: - break; + case START: + concurrentTasks += timeInfoSet.count(timeInfo); + break; + case FINISH: + concurrentTasks -= timeInfoSet.count(timeInfo); + break; + default: + break; } timeInfo.concurrentTasks = concurrentTasks; addToResult(vertexName, timeInfo.timestamp, timeInfo.concurrentTasks); @@ -132,7 +133,7 @@ public void analyze(DagInfo dagInfo) throws TezException { } private void addToResult(String vertexName, long currentTime, int concurrentTasks) { - String[] record = { currentTime + "", vertexName, concurrentTasks + "" }; + String[] record = {currentTime + "", vertexName, concurrentTasks + ""}; csvResult.addRecord(record); } diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java index 705c6e9cfb..1dcd6984de 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/TezAnalyzerBase.java @@ -54,7 +54,7 @@ public abstract class TezAnalyzerBase extends Configured implements Tool, Analyzer { private static final Logger LOG = LoggerFactory.getLogger(TezAnalyzerBase.class); - + private static final String EVENT_FILE_NAME = "eventFileName"; private static final String OUTPUT_DIR = "outputDir"; private static final String SAVE_RESULTS = "saveResults"; @@ -86,12 +86,12 @@ private static Options buildOptions() { .hasArg(false).isRequired(false).create(); Option eventFileNameOption = OptionBuilder.withArgName(EVENT_FILE_NAME).withLongOpt - (EVENT_FILE_NAME) + (EVENT_FILE_NAME) .withDescription("File with event data for the DAG").hasArg() .isRequired(false).create(); - + Option fromSimpleHistoryOption = OptionBuilder.withArgName(FROM_SIMPLE_HISTORY).withLongOpt - (FROM_SIMPLE_HISTORY) + (FROM_SIMPLE_HISTORY) .withDescription("Event data from Simple History logging. Must also specify event file") .isRequired(false).create(); @@ -101,7 +101,7 @@ private static Options buildOptions() { .isRequired(false).create(); Option help = OptionBuilder.withArgName(HELP).withLongOpt - (HELP) + (HELP) .withDescription("print help") .isRequired(false).create(); @@ -115,11 +115,11 @@ private static Options buildOptions() { opts.addOption(help); return opts; } - + protected String getOutputDir() { return outputDir; } - + private void printUsage() { System.err.println("Analyzer base options are"); Options options = buildOptions(); @@ -141,8 +141,8 @@ public int run(String[] args) throws Exception { return -1; } saveResults = cmdLine.hasOption(SAVE_RESULTS); - - if(cmdLine.hasOption(HELP)) { + + if (cmdLine.hasOption(HELP)) { printUsage(); return 0; } @@ -169,7 +169,7 @@ public int run(String[] args) throws Exception { } DagInfo dagInfo = null; - + if (files.isEmpty()) { if (cmdLine.hasOption(FROM_SIMPLE_HISTORY)) { System.err.println("Event file name must be specified when using simple history"); @@ -183,7 +183,7 @@ public int run(String[] args) throws Exception { } // using ATS - try to download directly - String[] importArgs = { "--dagId=" + dagId, "--downloadDir=" + outputDir }; + String[] importArgs = {"--dagId=" + dagId, "--downloadDir=" + outputDir}; int result = ATSImportTool.process(importArgs); if (result != 0) { @@ -196,7 +196,7 @@ public int run(String[] args) throws Exception { files.add(new File(outputDir + Path.SEPARATOR + dagId + ".zip")); } - + Preconditions.checkState(!files.isEmpty()); if (cmdLine.hasOption(FROM_SIMPLE_HISTORY)) { SimpleHistoryParser parser = new SimpleHistoryParser(files); diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/VertexLevelCriticalPathAnalyzer.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/VertexLevelCriticalPathAnalyzer.java index 78a4d41f38..2a19ddde73 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/VertexLevelCriticalPathAnalyzer.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/plugins/VertexLevelCriticalPathAnalyzer.java @@ -44,7 +44,7 @@ * Identify a set of vertices which fall in the critical path in a DAG. */ public class VertexLevelCriticalPathAnalyzer extends TezAnalyzerBase implements Analyzer { - private static final String[] headers = { "CriticalPath", "Score" }; + private static final String[] headers = {"CriticalPath", "Score"}; private final CSVResult csvResult; @@ -61,7 +61,8 @@ public VertexLevelCriticalPathAnalyzer(Configuration config) { this.dotFileLocation = config.get(DOT_FILE_DIR, DOT_FILE_DIR_DEFAULT); } - @Override public void analyze(DagInfo dagInfo) throws TezException { + @Override + public void analyze(DagInfo dagInfo) throws TezException { Map result = Maps.newLinkedHashMap(); getCriticalPath("", dagInfo.getVertices().get(dagInfo.getVertices().size() - 1), 0, result); @@ -112,7 +113,7 @@ private static Map sortByValues(Map result) { } private static void getCriticalPath(String predecessor, VertexInfo dest, long time, - Map result) { + Map result) { String destVertexName = (dest != null) ? (dest.getVertexName()) : ""; if (dest != null) { diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/utils/SVGUtils.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/utils/SVGUtils.java index 90acf3b0a0..44ec8ea04c 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/utils/SVGUtils.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/utils/SVGUtils.java @@ -41,7 +41,7 @@ public class SVGUtils { private static int MAX_DAG_RUNTIME = 0; private static final int SCREEN_WIDTH = 1800; - public SVGUtils() { + public SVGUtils() { } private int Y_MAX; @@ -63,82 +63,82 @@ public SVGUtils() { public static String getTimeStr(final long millis) { long minutes = TimeUnit.MILLISECONDS.toMinutes(millis) - - TimeUnit.HOURS.toMinutes(TimeUnit.MILLISECONDS.toHours(millis)); + - TimeUnit.HOURS.toMinutes(TimeUnit.MILLISECONDS.toHours(millis)); long hours = TimeUnit.MILLISECONDS.toHours(millis); StringBuilder b = new StringBuilder(); b.append(hours == 0 ? "" : String.valueOf(hours) + "h"); b.append(minutes == 0 ? "" : String.valueOf(minutes) + "m"); long seconds = millis - TimeUnit.MINUTES.toMillis( TimeUnit.MILLISECONDS.toMinutes(millis)); - b.append(secondFormat.format(seconds/1000.0) + "s"); - - return b.toString(); + b.append(secondFormat.format(seconds / 1000.0) + "s"); + + return b.toString(); } - + List svgLines = new LinkedList<>(); - + private final int addOffsetX(int x) { int xOff = x + X_BASE; X_MAX = Math.max(X_MAX, xOff); return xOff; } - + private final int addOffsetY(int y) { int yOff = y + Y_BASE; Y_MAX = Math.max(Y_MAX, yOff); return yOff; } - + private int scaleDown(int len) { return Math.round((len * 1.0f / MAX_DAG_RUNTIME) * SCREEN_WIDTH); } - - private void addRectStr(int x, int width, int y, int height, - String fillColor, String borderColor, float opacity, String title) { + + private void addRectStr(int x, int width, int y, int height, + String fillColor, String borderColor, float opacity, String title) { String rectStyle = "stroke: " + borderColor + "; fill: " + fillColor + "; opacity: " + opacity; String rectStr = "" - + " " + title +"" - + " "; - svgLines.add(rectStr); + + " y=\"" + addOffsetY(y) + "\"" + + " width=\"" + scaleDown(width) + "\"" + + " height=\"" + height + "\"" + + " style=\"" + rectStyle + "\"" + + " >" + + " " + title + "" + + " "; + svgLines.add(rectStr); } - + private void addTextStr(int x, int y, String text, String anchor, int size, String title, boolean italic) { - String textStyle = "text-anchor: " + anchor + "; font-style: " + (italic?"italic":"normal") + + String textStyle = "text-anchor: " + anchor + "; font-style: " + (italic ? "italic" : "normal") + "; font-size: " + size + "px;"; String textStr = "" + text - + " " + title +"" + + " " + title + "" + ""; svgLines.add(textStr); } - + private void addLineStr(int x1, int y1, int x2, int y2, String color, String title, int width) { String style = "stroke: " + color + "; stroke-width:" + width; String str = "" - + " " + title +"" - + " "; + + " y1=\"" + addOffsetY(y1) + "\"" + + " x2=\"" + addOffsetX(scaleDown(x2)) + "\"" + + " y2=\"" + addOffsetY(y2) + "\"" + + " style=\"" + style + "\"" + + " >" + + " " + title + "" + + " "; svgLines.add(str); } - + public void drawStep(CriticalPathStep step, long dagStartTime, int yOffset) { if (step.getType() != EntityType.ATTEMPT) { // draw initial vertex or final commit overhead StringBuilder title = new StringBuilder(); String text = null; if (step.getType() == EntityType.VERTEX_INIT) { - String vertex = step.getAttempt().getTaskInfo().getVertexInfo().getVertexName(); + String vertex = step.getAttempt().getTaskInfo().getVertexInfo().getVertexName(); text = vertex + " : Init"; title.append(text).append(TITLE_BR); } else { @@ -147,7 +147,7 @@ public void drawStep(CriticalPathStep step, long dagStartTime, int yOffset) { } title.append("Critical Path Dependency: " + step.getReason()).append(TITLE_BR); title.append( - "Critical Time: " + getTimeStr(step.getStopCriticalTime() - step.getStartCriticalTime())) + "Critical Time: " + getTimeStr(step.getStopCriticalTime() - step.getStartCriticalTime())) .append(""); title.append(Joiner.on(TITLE_BR).join(step.getNotes())); String titleStr = title.toString(); @@ -165,9 +165,9 @@ public void drawStep(CriticalPathStep step, long dagStartTime, int yOffset) { int startCriticalTimeInterval = (int) (step.getStartCriticalTime() - dagStartTime); int stopCriticalTimeInterval = (int) (step.getStopCriticalTime() - dagStartTime); int creationTimeInterval = (int) (attempt.getCreationTime() - dagStartTime); - int allocationTimeInterval = attempt.getAllocationTime() > 0 ? + int allocationTimeInterval = attempt.getAllocationTime() > 0 ? (int) (attempt.getAllocationTime() - dagStartTime) : 0; - int launchTimeInterval = attempt.getStartTime() > 0 ? + int launchTimeInterval = attempt.getStartTime() > 0 ? (int) (attempt.getStartTime() - dagStartTime) : 0; int finishTimeInterval = (int) (attempt.getFinishTime() - dagStartTime); LOG.debug(attempt.getTaskAttemptId() + " " + creationTimeInterval + " " @@ -178,7 +178,7 @@ public void drawStep(CriticalPathStep step, long dagStartTime, int yOffset) { title.append("Critical Path Dependency: " + step.getReason()).append(TITLE_BR); title.append("Completion Status: " + attempt.getDetailedStatus()).append(TITLE_BR); title.append( - "Critical Time Contribution: " + + "Critical Time Contribution: " + getTimeStr(step.getStopCriticalTime() - step.getStartCriticalTime())).append(TITLE_BR); title.append("Critical start at: " + getTimeStr(startCriticalTimeInterval)).append(TITLE_BR); title.append("Critical stop at: " + getTimeStr(stopCriticalTimeInterval)).append(TITLE_BR); @@ -201,22 +201,22 @@ public void drawStep(CriticalPathStep step, long dagStartTime, int yOffset) { if (launchTimeInterval > 0) { addRectStr(allocationTimeInterval, launchTimeInterval - allocationTimeInterval, yOffset * STEP_GAP, STEP_GAP, LAUNCH_OVERHEAD_COLOR, BORDER_COLOR, RECT_OPACITY, - titleStr); + titleStr); addRectStr(launchTimeInterval, finishTimeInterval - launchTimeInterval, yOffset * STEP_GAP, STEP_GAP, RUNTIME_COLOR, BORDER_COLOR, RECT_OPACITY, titleStr); } else { // no launch - so allocate to finish drawn - ended while launching addRectStr(allocationTimeInterval, finishTimeInterval - allocationTimeInterval, yOffset * STEP_GAP, - STEP_GAP, LAUNCH_OVERHEAD_COLOR, BORDER_COLOR, RECT_OPACITY, titleStr); + STEP_GAP, LAUNCH_OVERHEAD_COLOR, BORDER_COLOR, RECT_OPACITY, titleStr); } } else { // no allocation - so create to finish drawn - ended while allocating addRectStr(creationTimeInterval, finishTimeInterval - creationTimeInterval, yOffset * STEP_GAP, - STEP_GAP, ALLOCATION_OVERHEAD_COLOR, BORDER_COLOR, RECT_OPACITY, titleStr); + STEP_GAP, ALLOCATION_OVERHEAD_COLOR, BORDER_COLOR, RECT_OPACITY, titleStr); } addTextStr((finishTimeInterval + creationTimeInterval) / 2, - (yOffset * STEP_GAP + STEP_GAP / 2), attempt.getShortName(), "middle", TEXT_SIZE, + (yOffset * STEP_GAP + STEP_GAP / 2), attempt.getShortName(), "middle", TEXT_SIZE, titleStr, !attempt.isSucceeded()); } } @@ -227,17 +227,17 @@ private void drawCritical(DagInfo dagInfo, List criticalPath) int dagFinishTimeInterval = (int) (dagInfo.getFinishTime() - dagStartTime); if (dagInfo.getFinishTime() <= 0) { // AM crashed. no dag finish time written - dagFinishTimeInterval =(int) (criticalPath.get(criticalPath.size()-1).getStopCriticalTime() + dagFinishTimeInterval = (int) (criticalPath.get(criticalPath.size() - 1).getStopCriticalTime() - dagStartTime); } MAX_DAG_RUNTIME = dagFinishTimeInterval; - + // draw grid addLineStr(dagStartTimeInterval, 0, dagFinishTimeInterval, 0, BORDER_COLOR, "", TICK); - int yGrid = (criticalPath.size() + 2)*STEP_GAP; - for (int i=0; i<11; ++i) { - int x = Math.round(((dagFinishTimeInterval - dagStartTimeInterval)/10.0f)*i); - addLineStr(x, 0, x, yGrid, BORDER_COLOR, "", TICK); + int yGrid = (criticalPath.size() + 2) * STEP_GAP; + for (int i = 0; i < 11; ++i) { + int x = Math.round(((dagFinishTimeInterval - dagStartTimeInterval) / 10.0f) * i); + addLineStr(x, 0, x, yGrid, BORDER_COLOR, "", TICK); addTextStr(x, 0, getTimeStr(x), "left", TEXT_SIZE, "", false); } addLineStr(dagStartTimeInterval, yGrid, dagFinishTimeInterval, yGrid, BORDER_COLOR, "", TICK); @@ -246,62 +246,62 @@ private void drawCritical(DagInfo dagInfo, List criticalPath) TEXT_SIZE, "", false); // draw steps - for (int i=1; i<=criticalPath.size(); ++i) { - CriticalPathStep step = criticalPath.get(i-1); - drawStep(step, dagStartTime, i); + for (int i = 1; i <= criticalPath.size(); ++i) { + CriticalPathStep step = criticalPath.get(i - 1); + drawStep(step, dagStartTime, i); } - + // draw critical path on top - for (int i=1; i<=criticalPath.size(); ++i) { - CriticalPathStep step = criticalPath.get(i-1); - boolean isLast = i == criticalPath.size(); - + for (int i = 1; i <= criticalPath.size(); ++i) { + CriticalPathStep step = criticalPath.get(i - 1); + boolean isLast = i == criticalPath.size(); + // draw critical path for step int startCriticalTimeInterval = (int) (step.getStartCriticalTime() - dagStartTime); int stopCriticalTimeInterval = (int) (step.getStopCriticalTime() - dagStartTime); addLineStr(startCriticalTimeInterval, (i + 1) * STEP_GAP, stopCriticalTimeInterval, - (i + 1) * STEP_GAP, CRITICAL_COLOR, "Critical Time " + step.getAttempt().getShortName(), TICK*5); - + (i + 1) * STEP_GAP, CRITICAL_COLOR, "Critical Time " + step.getAttempt().getShortName(), TICK * 5); + if (isLast) { // last step. add commit overhead int stepStopCriticalTimeInterval = (int) (step.getStopCriticalTime() - dagStartTime); addLineStr(stepStopCriticalTimeInterval, (i + 1) * STEP_GAP, dagFinishTimeInterval, (i + 1) * STEP_GAP, CRITICAL_COLOR, - "Critical Time " + step.getAttempt().getTaskInfo().getVertexInfo().getVertexName(), TICK*5); + "Critical Time " + step.getAttempt().getTaskInfo().getVertexInfo().getVertexName(), TICK * 5); } else { // connect to next step in critical path addLineStr(stopCriticalTimeInterval, (i + 1) * STEP_GAP, stopCriticalTimeInterval, - (i + 2) * STEP_GAP, CRITICAL_COLOR, "Critical Time " + step.getAttempt().getShortName(), TICK*5); + (i + 2) * STEP_GAP, CRITICAL_COLOR, "Critical Time " + step.getAttempt().getShortName(), TICK * 5); } } - + // draw legend int legendX = 0; int legendY = (criticalPath.size() + 2) * STEP_GAP; - int legendWidth = dagFinishTimeInterval/5; - - addRectStr(legendX, legendWidth, legendY, STEP_GAP/2, VERTEX_INIT_COMMIT_COLOR, BORDER_COLOR, RECT_OPACITY, ""); - addTextStr(legendX, legendY + STEP_GAP/3, "Vertex Init/Commit Overhead", "left", TEXT_SIZE, "", false); - legendY += STEP_GAP/2; - addRectStr(legendX, legendWidth, legendY, STEP_GAP/2, ALLOCATION_OVERHEAD_COLOR, BORDER_COLOR, RECT_OPACITY, ""); - addTextStr(legendX, legendY + STEP_GAP/3, "Task Allocation Overhead", "left", TEXT_SIZE, "", false); - legendY += STEP_GAP/2; - addRectStr(legendX, legendWidth, legendY, STEP_GAP/2, LAUNCH_OVERHEAD_COLOR, BORDER_COLOR, RECT_OPACITY, ""); - addTextStr(legendX, legendY + STEP_GAP/3, "Task Launch Overhead", "left", TEXT_SIZE, "", false); - legendY += STEP_GAP/2; - addRectStr(legendX, legendWidth, legendY, STEP_GAP/2, RUNTIME_COLOR, BORDER_COLOR, RECT_OPACITY, ""); - addTextStr(legendX, legendY + STEP_GAP/3, "Task Execution Time", "left", TEXT_SIZE, "", false); - - Y_MAX += Y_BASE*2; - X_MAX += X_BASE*2; + int legendWidth = dagFinishTimeInterval / 5; + + addRectStr(legendX, legendWidth, legendY, STEP_GAP / 2, VERTEX_INIT_COMMIT_COLOR, BORDER_COLOR, RECT_OPACITY, ""); + addTextStr(legendX, legendY + STEP_GAP / 3, "Vertex Init/Commit Overhead", "left", TEXT_SIZE, "", false); + legendY += STEP_GAP / 2; + addRectStr(legendX, legendWidth, legendY, STEP_GAP / 2, ALLOCATION_OVERHEAD_COLOR, BORDER_COLOR, RECT_OPACITY, ""); + addTextStr(legendX, legendY + STEP_GAP / 3, "Task Allocation Overhead", "left", TEXT_SIZE, "", false); + legendY += STEP_GAP / 2; + addRectStr(legendX, legendWidth, legendY, STEP_GAP / 2, LAUNCH_OVERHEAD_COLOR, BORDER_COLOR, RECT_OPACITY, ""); + addTextStr(legendX, legendY + STEP_GAP / 3, "Task Launch Overhead", "left", TEXT_SIZE, "", false); + legendY += STEP_GAP / 2; + addRectStr(legendX, legendWidth, legendY, STEP_GAP / 2, RUNTIME_COLOR, BORDER_COLOR, RECT_OPACITY, ""); + addTextStr(legendX, legendY + STEP_GAP / 3, "Task Execution Time", "left", TEXT_SIZE, "", false); + + Y_MAX += Y_BASE * 2; + X_MAX += X_BASE * 2; } - - public void saveCriticalPathAsSVG(DagInfo dagInfo, - String fileName, List criticalPath) { + + public void saveCriticalPathAsSVG(DagInfo dagInfo, + String fileName, List criticalPath) { drawCritical(dagInfo, criticalPath); saveFileStr(fileName); } - + private void saveFileStr(String fileName) { String header = " " + " " + + "width=\"" + X_MAX + "\"> " + "