Skip to content

Commit 56fd79f

Browse files
tomicoolertomicooler
committed
TestCapacityScheduler: found one more app related test
Change-Id: I4afdc69a342debc770c1fc8b43ffb5e3088e74b6
1 parent 53ad1b1 commit 56fd79f

2 files changed

Lines changed: 152 additions & 160 deletions

File tree

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java‎

Lines changed: 0 additions & 160 deletions
Original file line numberDiff line numberDiff line change
@@ -2663,166 +2663,6 @@ public void testContainerAllocationLocalitySkipped() throws Exception {
26632663
ContainerAllocation.QUEUE_SKIPPED.getAllocationState());
26642664
}
26652665

2666-
@Test
2667-
public void testMoveAppWithActiveUsersWithOnlyPendingApps() throws Exception {
2668-
2669-
YarnConfiguration conf = new YarnConfiguration();
2670-
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
2671-
ResourceScheduler.class);
2672-
2673-
CapacitySchedulerConfiguration newConf =
2674-
new CapacitySchedulerConfiguration(conf);
2675-
2676-
// Define top-level queues
2677-
newConf.setQueues(CapacitySchedulerConfiguration.ROOT,
2678-
new String[] { "a", "b" });
2679-
2680-
newConf.setCapacity(A, 50);
2681-
newConf.setCapacity(B, 50);
2682-
2683-
// Define 2nd-level queues
2684-
newConf.setQueues(A, new String[] { "a1" });
2685-
newConf.setCapacity(A1, 100);
2686-
newConf.setUserLimitFactor(A1, 2.0f);
2687-
newConf.setMaximumAMResourcePercentPerPartition(A1, "", 0.1f);
2688-
2689-
newConf.setQueues(B, new String[] { "b1" });
2690-
newConf.setCapacity(B1, 100);
2691-
newConf.setUserLimitFactor(B1, 2.0f);
2692-
2693-
LOG.info("Setup top-level queues a and b");
2694-
2695-
MockRM rm = new MockRM(newConf);
2696-
rm.start();
2697-
2698-
CapacityScheduler scheduler =
2699-
(CapacityScheduler) rm.getResourceScheduler();
2700-
2701-
MockNM nm1 = rm.registerNode("h1:1234", 16 * GB);
2702-
2703-
// submit an app
2704-
MockRMAppSubmissionData data3 =
2705-
MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
2706-
.withAppName("test-move-1")
2707-
.withUser("u1")
2708-
.withAcls(null)
2709-
.withQueue("a1")
2710-
.withUnmanagedAM(false)
2711-
.build();
2712-
RMApp app = MockRMAppSubmitter.submit(rm, data3);
2713-
MockAM am1 = MockRM.launchAndRegisterAM(app, rm, nm1);
2714-
2715-
ApplicationAttemptId appAttemptId =
2716-
rm.getApplicationReport(app.getApplicationId())
2717-
.getCurrentApplicationAttemptId();
2718-
2719-
MockRMAppSubmissionData data2 =
2720-
MockRMAppSubmissionData.Builder.createWithMemory(1 * GB, rm)
2721-
.withAppName("app")
2722-
.withUser("u2")
2723-
.withAcls(null)
2724-
.withQueue("a1")
2725-
.withUnmanagedAM(false)
2726-
.build();
2727-
RMApp app2 = MockRMAppSubmitter.submit(rm, data2);
2728-
MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1);
2729-
2730-
MockRMAppSubmissionData data1 =
2731-
MockRMAppSubmissionData.Builder.createWithMemory(1 * GB, rm)
2732-
.withAppName("app")
2733-
.withUser("u3")
2734-
.withAcls(null)
2735-
.withQueue("a1")
2736-
.withUnmanagedAM(false)
2737-
.build();
2738-
RMApp app3 = MockRMAppSubmitter.submit(rm, data1);
2739-
2740-
MockRMAppSubmissionData data =
2741-
MockRMAppSubmissionData.Builder.createWithMemory(1 * GB, rm)
2742-
.withAppName("app")
2743-
.withUser("u4")
2744-
.withAcls(null)
2745-
.withQueue("a1")
2746-
.withUnmanagedAM(false)
2747-
.build();
2748-
RMApp app4 = MockRMAppSubmitter.submit(rm, data);
2749-
2750-
// Each application asks 50 * 1GB containers
2751-
am1.allocate("*", 1 * GB, 50, null);
2752-
am2.allocate("*", 1 * GB, 50, null);
2753-
2754-
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
2755-
RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
2756-
2757-
// check preconditions
2758-
List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1");
2759-
assertEquals(4, appsInA1.size());
2760-
String queue =
2761-
scheduler.getApplicationAttempt(appsInA1.get(0)).getQueue()
2762-
.getQueueName();
2763-
Assert.assertEquals("a1", queue);
2764-
2765-
List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a");
2766-
assertTrue(appsInA.contains(appAttemptId));
2767-
assertEquals(4, appsInA.size());
2768-
2769-
List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root");
2770-
assertTrue(appsInRoot.contains(appAttemptId));
2771-
assertEquals(4, appsInRoot.size());
2772-
2773-
List<ApplicationAttemptId> appsInB1 = scheduler.getAppsInQueue("b1");
2774-
assertTrue(appsInB1.isEmpty());
2775-
2776-
List<ApplicationAttemptId> appsInB = scheduler.getAppsInQueue("b");
2777-
assertTrue(appsInB.isEmpty());
2778-
2779-
UsersManager um =
2780-
(UsersManager) scheduler.getQueue("a1").getAbstractUsersManager();
2781-
2782-
assertEquals(4, um.getNumActiveUsers());
2783-
assertEquals(2, um.getNumActiveUsersWithOnlyPendingApps());
2784-
2785-
// now move the app
2786-
scheduler.moveAllApps("a1", "b1");
2787-
2788-
//Triggering this event so that user limit computation can
2789-
//happen again
2790-
for (int i = 0; i < 10; i++) {
2791-
cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
2792-
Thread.sleep(500);
2793-
}
2794-
2795-
// check postconditions
2796-
appsInB1 = scheduler.getAppsInQueue("b1");
2797-
2798-
assertEquals(4, appsInB1.size());
2799-
queue =
2800-
scheduler.getApplicationAttempt(appsInB1.get(0)).getQueue()
2801-
.getQueueName();
2802-
Assert.assertEquals("b1", queue);
2803-
2804-
appsInB = scheduler.getAppsInQueue("b");
2805-
assertTrue(appsInB.contains(appAttemptId));
2806-
assertEquals(4, appsInB.size());
2807-
2808-
appsInRoot = scheduler.getAppsInQueue("root");
2809-
assertTrue(appsInRoot.contains(appAttemptId));
2810-
assertEquals(4, appsInRoot.size());
2811-
2812-
List<ApplicationAttemptId> oldAppsInA1 = scheduler.getAppsInQueue("a1");
2813-
assertEquals(0, oldAppsInA1.size());
2814-
2815-
UsersManager um_b1 =
2816-
(UsersManager) scheduler.getQueue("b1").getAbstractUsersManager();
2817-
2818-
assertEquals(2, um_b1.getNumActiveUsers());
2819-
assertEquals(2, um_b1.getNumActiveUsersWithOnlyPendingApps());
2820-
2821-
appsInB1 = scheduler.getAppsInQueue("b1");
2822-
assertEquals(4, appsInB1.size());
2823-
rm.close();
2824-
}
2825-
28262666
@Test
28272667
public void testCSQueueMetrics() throws Exception {
28282668

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@
7676
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
7777
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
7878
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
79+
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
7980
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
8081
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo;
8182
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerLeafQueueInfo;
@@ -1102,6 +1103,157 @@ public void testMoveAllAppsInvalidSource() throws Exception {
11021103
rm.stop();
11031104
}
11041105

1106+
@Test
1107+
public void testMoveAppWithActiveUsersWithOnlyPendingApps() throws Exception {
1108+
YarnConfiguration conf = new YarnConfiguration();
1109+
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
1110+
ResourceScheduler.class);
1111+
1112+
CapacitySchedulerConfiguration newConf =
1113+
new CapacitySchedulerConfiguration(conf);
1114+
1115+
// Define top-level queues
1116+
newConf.setQueues(CapacitySchedulerConfiguration.ROOT,
1117+
new String[]{"a", "b"});
1118+
1119+
newConf.setCapacity(A, 50);
1120+
newConf.setCapacity(B, 50);
1121+
1122+
// Define 2nd-level queues
1123+
newConf.setQueues(A, new String[]{"a1"});
1124+
newConf.setCapacity(A1, 100);
1125+
newConf.setUserLimitFactor(A1, 2.0f);
1126+
newConf.setMaximumAMResourcePercentPerPartition(A1, "", 0.1f);
1127+
1128+
newConf.setQueues(B, new String[]{"b1"});
1129+
newConf.setCapacity(B1, 100);
1130+
newConf.setUserLimitFactor(B1, 2.0f);
1131+
1132+
MockRM rm = new MockRM(newConf);
1133+
rm.start();
1134+
1135+
CapacityScheduler scheduler =
1136+
(CapacityScheduler) rm.getResourceScheduler();
1137+
1138+
MockNM nm1 = rm.registerNode("h1:1234", 16 * GB);
1139+
1140+
// submit an app
1141+
MockRMAppSubmissionData data3 =
1142+
MockRMAppSubmissionData.Builder.createWithMemory(GB, rm)
1143+
.withAppName("test-move-1")
1144+
.withUser("u1")
1145+
.withAcls(null)
1146+
.withQueue("a1")
1147+
.withUnmanagedAM(false)
1148+
.build();
1149+
RMApp app = MockRMAppSubmitter.submit(rm, data3);
1150+
MockAM am1 = MockRM.launchAndRegisterAM(app, rm, nm1);
1151+
1152+
ApplicationAttemptId appAttemptId =
1153+
rm.getApplicationReport(app.getApplicationId())
1154+
.getCurrentApplicationAttemptId();
1155+
1156+
MockRMAppSubmissionData data2 =
1157+
MockRMAppSubmissionData.Builder.createWithMemory(1 * GB, rm)
1158+
.withAppName("app")
1159+
.withUser("u2")
1160+
.withAcls(null)
1161+
.withQueue("a1")
1162+
.withUnmanagedAM(false)
1163+
.build();
1164+
RMApp app2 = MockRMAppSubmitter.submit(rm, data2);
1165+
MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1);
1166+
1167+
MockRMAppSubmissionData data1 =
1168+
MockRMAppSubmissionData.Builder.createWithMemory(1 * GB, rm)
1169+
.withAppName("app")
1170+
.withUser("u3")
1171+
.withAcls(null)
1172+
.withQueue("a1")
1173+
.withUnmanagedAM(false)
1174+
.build();
1175+
RMApp app3 = MockRMAppSubmitter.submit(rm, data1);
1176+
1177+
MockRMAppSubmissionData data =
1178+
MockRMAppSubmissionData.Builder.createWithMemory(1 * GB, rm)
1179+
.withAppName("app")
1180+
.withUser("u4")
1181+
.withAcls(null)
1182+
.withQueue("a1")
1183+
.withUnmanagedAM(false)
1184+
.build();
1185+
RMApp app4 = MockRMAppSubmitter.submit(rm, data);
1186+
1187+
// Each application asks 50 * 1GB containers
1188+
am1.allocate("*", 1 * GB, 50, null);
1189+
am2.allocate("*", 1 * GB, 50, null);
1190+
1191+
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
1192+
RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
1193+
1194+
// check preconditions
1195+
assertApps(scheduler, "root",
1196+
app3.getCurrentAppAttempt().getAppAttemptId(),
1197+
app4.getCurrentAppAttempt().getAppAttemptId(),
1198+
appAttemptId,
1199+
app2.getCurrentAppAttempt().getAppAttemptId());
1200+
assertApps(scheduler, "a",
1201+
app3.getCurrentAppAttempt().getAppAttemptId(),
1202+
app4.getCurrentAppAttempt().getAppAttemptId(),
1203+
appAttemptId,
1204+
app2.getCurrentAppAttempt().getAppAttemptId());
1205+
assertApps(scheduler, "a1",
1206+
app3.getCurrentAppAttempt().getAppAttemptId(),
1207+
app4.getCurrentAppAttempt().getAppAttemptId(),
1208+
appAttemptId,
1209+
app2.getCurrentAppAttempt().getAppAttemptId());
1210+
assertApps(scheduler, "b");
1211+
assertApps(scheduler, "b1");
1212+
1213+
UsersManager um =
1214+
(UsersManager) scheduler.getQueue("a1").getAbstractUsersManager();
1215+
1216+
assertEquals(4, um.getNumActiveUsers());
1217+
assertEquals(2, um.getNumActiveUsersWithOnlyPendingApps());
1218+
1219+
// now move the app
1220+
scheduler.moveAllApps("a1", "b1");
1221+
1222+
//Triggering this event so that user limit computation can
1223+
//happen again
1224+
for (int i = 0; i < 10; i++) {
1225+
cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
1226+
Thread.sleep(500);
1227+
}
1228+
1229+
// check post conditions
1230+
assertApps(scheduler, "root",
1231+
appAttemptId,
1232+
app2.getCurrentAppAttempt().getAppAttemptId(),
1233+
app3.getCurrentAppAttempt().getAppAttemptId(),
1234+
app4.getCurrentAppAttempt().getAppAttemptId());
1235+
assertApps(scheduler, "a");
1236+
assertApps(scheduler, "a1");
1237+
assertApps(scheduler, "b",
1238+
appAttemptId,
1239+
app2.getCurrentAppAttempt().getAppAttemptId(),
1240+
app3.getCurrentAppAttempt().getAppAttemptId(),
1241+
app4.getCurrentAppAttempt().getAppAttemptId());
1242+
assertApps(scheduler, "b1",
1243+
appAttemptId,
1244+
app2.getCurrentAppAttempt().getAppAttemptId(),
1245+
app3.getCurrentAppAttempt().getAppAttemptId(),
1246+
app4.getCurrentAppAttempt().getAppAttemptId());
1247+
1248+
UsersManager umB1 =
1249+
(UsersManager) scheduler.getQueue("b1").getAbstractUsersManager();
1250+
1251+
assertEquals(2, umB1.getNumActiveUsers());
1252+
assertEquals(2, umB1.getNumActiveUsersWithOnlyPendingApps());
1253+
1254+
rm.close();
1255+
}
1256+
11051257
@Test(timeout = 60000)
11061258
public void testMoveAttemptNotAdded() throws Exception {
11071259
Configuration conf = new Configuration();

0 commit comments

Comments
 (0)