YARN-5025. Added and updated container management protocols for container relocation

Author: szape

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java

@@ -31,6 +31,8 @@
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.hadoop.yarn.api.protocolrecords.CommitResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerLaunchContextRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerLaunchContextResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest;
@@ -481,6 +483,13 @@ public ResourceLocalizationResponse localize(
         ResourceLocalizationRequest request) throws YarnException, IOException {
       return null;
     }
+  
+    @Override
+    public GetContainerLaunchContextResponse getContainerLaunchContext(
+        GetContainerLaunchContextRequest request) throws YarnException, IOException {
+      throw new UnsupportedOperationException("getting the container launch context is not " +
+          "supported for this implementation of ContainerManagementProtocol");
+    }
 
     @Override
     public ReInitializeContainerResponse reInitializeContainer(

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java

@@ -47,10 +47,12 @@
 import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils;
 import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.CommitResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerLaunchContextRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerLaunchContextResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest;
@@ -70,7 +72,8 @@
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.Token;
-import org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData;
+import org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy
+    .ContainerManagementProtocolProxyData;
 import org.apache.hadoop.yarn.event.Event;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -486,6 +489,13 @@ public ResourceLocalizationResponse localize(
         ResourceLocalizationRequest request) throws YarnException, IOException {
       return null;
     }
+  
+    @Override
+    public GetContainerLaunchContextResponse getContainerLaunchContext(
+        GetContainerLaunchContextRequest request) throws YarnException, IOException {
+      throw new UnsupportedOperationException("getting the container launch context is not " +
+          "supported for this implementation of ContainerManagementProtocol");
+    }
 
     @Override
     public ReInitializeContainerResponse reInitializeContainer(

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java

@@ -24,6 +24,8 @@
 import org.apache.hadoop.classification.InterfaceStability.Stable;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.protocolrecords.CommitResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerLaunchContextRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerLaunchContextResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
@@ -51,7 +53,7 @@
 /**
  * <p>The protocol between an <code>ApplicationMaster</code> and a 
  * <code>NodeManager</code> to start/stop and increase resource of containers
- * and to get status of running containers.</p>
+ * and to get status and launch context of running containers.</p>
  *
  * <p>If security is enabled the <code>NodeManager</code> verifies that the
  * <code>ApplicationMaster</code> has truly been allocated the container
@@ -220,6 +222,22 @@ SignalContainerResponse signalToContainer(SignalContainerRequest request)
   @Unstable
   ResourceLocalizationResponse localize(ResourceLocalizationRequest request)
     throws YarnException, IOException;
+  
+  /**
+   * Gets container launch context for a container specified in
+   * {@link GetContainerLaunchContextRequest}.
+   * This protocol is only used by the container relocation logic, between node managers, to
+   * transfer the launch context of the container to be relocated to the target node.
+   *
+   * @param request specifies the id of the container for which the launch context is requested
+   * @return Response that contains the requested container launch context
+   * @throws YarnException
+   * @throws IOException
+   */
+  @Public
+  @Stable
+  GetContainerLaunchContextResponse getContainerLaunchContext(
+      GetContainerLaunchContextRequest request) throws YarnException, IOException;
 
   /**
    * ReInitialize the Container with a new Launch Context.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerLaunchContextRequest.java

@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.protocolrecords;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>The request sent by one <code>NodeManager</code> to another
+ * <code>NodeManager</code> in order to get the launch context of the container with id
+ * <em>containerId</em>.</p>
+ *
+ * <p>This request is used only for container relocation, where the launch context of
+ * the origin container is necessary for launching the relocated container on the target node.</p>
+ *
+ * @see ContainerManagementProtocol#getContainerLaunchContext(GetContainerLaunchContextRequest)
+ */
+public abstract class GetContainerLaunchContextRequest {
+  
+  @Public
+  @Stable
+  public static GetContainerLaunchContextRequest newInstance(ContainerId containerId) {
+    GetContainerLaunchContextRequest request =
+        Records.newRecord(GetContainerLaunchContextRequest.class);
+    request.setContainerId(containerId);
+    return request;
+  }
+  
+  /**
+   * Gets the container id for which the launch context is requested.
+   * @return the container id for which the launch context is requested
+   */
+  @Public
+  @Stable
+  public abstract ContainerId getContainerId();
+  
+  /**
+   * Sets the container id for which the launch context is requested.
+   * @param containerId the container id for which the launch context is requested
+   */
+  @Public
+  @Stable
+  public abstract void setContainerId(ContainerId containerId);
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerLaunchContextResponse.java

@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.protocolrecords;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>The response for a container launch context request sent between
+ * <code>NodeManager</code>s. It contains the requested container launch context.</p>
+ *
+ * <p>This request is used only for container relocation, where a launch context
+ * of the origin container is necessary for launching the relocated container on
+ * the target node.</p>
+ */
+public abstract class GetContainerLaunchContextResponse {
+  
+  @Public
+  @Stable
+  public static GetContainerLaunchContextResponse newInstance(ContainerLaunchContext
+      containerLaunchContext) {
+    GetContainerLaunchContextResponse request =
+        Records.newRecord(GetContainerLaunchContextResponse.class);
+    request.setContainerLaunchContext(containerLaunchContext);
+    return request;
+  }
+  
+  /**
+   * Gets the launch context of the requested container.
+   * @return the launch context of the requested container
+   */
+  @Public
+  @Stable
+  public abstract ContainerLaunchContext getContainerLaunchContext();
+  
+  /**
+   * Sets the launch context of the requested container.
+   * @param containerLaunchContext the launch context of the requested container
+   */
+  @Public
+  @Stable
+  public abstract void setContainerLaunchContext(ContainerLaunchContext containerLaunchContext);
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerRequest.java

@@ -20,9 +20,12 @@
 
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
+import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.NMToken;
+import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Token;
 import org.apache.hadoop.yarn.util.Records;
 
@@ -36,6 +39,15 @@
  * necessary binaries/jar/shared-objects etc. via the 
  * {@link ContainerLaunchContext}.</p>
  *
+ * <p>The <em>isMove</em> flag tells whether this request corresponds to a container
+ * relocation. If true, then no container launch context is needed as it will be
+ * transferred directly from the origin node manager to the target node manager.
+ * The container to be relocated is identified by the <em>originNodeId</em> and
+ * <em>originContainerId</em>.
+ * The originNMToken is sent along so that the origin container can be shut down
+ * by the target node manager.
+ * </p>
+ *
  * @see ContainerManagementProtocol#startContainers(StartContainersRequest)
  */
 @Public
@@ -49,6 +61,21 @@ public static StartContainerRequest newInstance(
         Records.newRecord(StartContainerRequest.class);
     request.setContainerLaunchContext(context);
     request.setContainerToken(container);
+    request.setIsMove(false);
+    return request;
+  }
+  
+  @Public
+  @Stable
+  public static StartContainerRequest newInstance(
+      Token container,  ContainerId originContainerId, NodeId originNodeId, Token originNMToken) {
+    StartContainerRequest request =
+        Records.newRecord(StartContainerRequest.class);
+    request.setContainerToken(container);
+    request.setIsMove(true);
+    request.setOriginContainerId(originContainerId);
+    request.setOriginNodeId(originNodeId);
+    request.setOriginNMToken(originNMToken);
     return request;
   }
 
@@ -91,4 +118,91 @@ public static StartContainerRequest newInstance(
   @Public
   @Stable
   public abstract void setContainerToken(Token container);
+  
+  /**
+   * Gets whether this start container request corresponds to a container relocation.
+   * @return whether this start container request corresponds to a container relocation
+   */
+  @Public
+  @Unstable
+  public abstract boolean getIsMove();
+  
+  /**
+   * Sets whether this start container request corresponds to a container relocation.
+   * @param isMove whether this start container request corresponds to a container
+   *               relocation
+   */
+  @Public
+  @Unstable
+  public abstract void setIsMove(boolean isMove);
+  
+  /**
+   * Gets the origin container id for this start container request.
+   * The origin container id is set if and only if this start container request
+   * corresponds to a container relocation. It identifies the container that should
+   * be relocated.
+   *
+   * @return the origin container id for this start container request
+   */
+  @Public
+  @Unstable
+  public abstract ContainerId getOriginContainerId();
+  
+  /**
+   * Sets the origin container id for this start container request.
+   * The origin container id should be set if and only if this start container request
+   * corresponds to a container relocation. It identifies the container that should
+   * be relocated.
+   *
+   * @param originContainerId the origin container id for this start container request
+   */
+  @Public
+  @Unstable
+  public abstract void setOriginContainerId(ContainerId originContainerId);
+  
+  /**
+   * Gets the origin node id for this start container request.
+   * The origin node id is set if and only if this resource start container corresponds
+   * to a container relocation. It identifies the node of the container that should
+   * be relocated.
+   *
+   * @return the origin node id for this start container request
+   */
+  @Public
+  @Unstable
+  public abstract NodeId getOriginNodeId();
+  
+  /**
+   * Sets the origin node id for this start container request.
+   * The origin node id should be set if and only if this start container request
+   * corresponds to a container relocation. It identifies the node of the container
+   * that should be relocated.
+   *
+   * @param originNodeId the origin node id for this start container request
+   */
+  @Public
+  @Unstable
+  public abstract void setOriginNodeId(NodeId originNodeId);
+  
+  /**
+   * Gets the security token for the origin node.
+   * The origin NM token is set if and only if this start container request corresponds
+   * to a container relocation. It is used for shutting down the origin container.
+   *
+   * @return the security token for the origin node
+   */
+  @Public
+  @Unstable
+  public abstract Token getOriginNMToken();
+  
+  /**
+   * Sets the security token for the origin node.
+   * The origin NM token should be set if and only if this start container request corresponds
+   * to a container relocation. It is used for shutting down the origin container.
+   *
+   * @param originNMToken the security token for the origin node
+   */
+  @Public
+  @Unstable
+  public abstract void setOriginNMToken(Token originNMToken);
 }