Switched to js solution and updated docs

Author: ajbozarth

core/src/main/resources/org/apache/spark/ui/static/historypage.js

@@ -15,6 +15,12 @@
  * limitations under the License.
  */
 
+var appLimit = -1;
+
+function setAppLimit(val) {
+    appLimit = val;
+}
+
 function makeIdNumeric(id) {
   var strs = id.split("_");
   if (strs.length < 3) {
@@ -89,7 +95,7 @@ $(document).ready(function() {
     requestedIncomplete = getParameterByName("showIncomplete", searchString);
     requestedIncomplete = (requestedIncomplete == "true" ? true : false);
 
-    $.getJSON("api/v1/applications", function(response,status,jqXHR) {
+    $.getJSON("api/v1/applications?limit=" + appLimit, function(response,status,jqXHR) {
       var array = [];
       var hasMultipleAttempts = false;
       for (i in response) {

core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala

@@ -45,7 +45,8 @@ private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("")
               <script src={UIUtils.prependBaseUri("/static/dataTables.rowsGroup.js")}></script> ++
                 <div id="history-summary" class="span12 pagination"></div> ++
                 <script src={UIUtils.prependBaseUri("/static/utils.js")}></script> ++
-                <script src={UIUtils.prependBaseUri("/static/historypage.js")}></script>
+                <script src={UIUtils.prependBaseUri("/static/historypage.js")}></script> ++
+                <script>setAppLimit({parent.maxApplications})</script>
             } else if (requestedIncomplete) {
               <h4>No incomplete applications found!</h4>
             } else {

core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala

@@ -56,8 +56,8 @@ class HistoryServer(
   // How many applications to retain
   private val retainedApplications = conf.getInt("spark.history.retainedApplications", 50)
 
-  // How many applications are available
-  private val maxApplications = conf.get(HISTORY_MAX_APPS);
+  // How many applications the summary ui displays
+  private[history] val maxApplications = conf.get(HISTORY_UI_MAX_APPS);
 
   // application
   private val appCache = new ApplicationCache(this, retainedApplications, new SystemClock())
@@ -175,7 +175,7 @@ class HistoryServer(
    * @return List of all known applications.
    */
   def getApplicationList(): Iterable[ApplicationHistoryInfo] = {
-    provider.getListing().take(maxApplications)
+    provider.getListing()
   }
 
   def getApplicationInfoList: Iterator[ApplicationInfo] = {

core/src/main/scala/org/apache/spark/internal/config/package.scala

@@ -120,7 +120,7 @@ package object config {
     .intConf
     .createWithDefault(100000)
 
-  // To limit how many applications are available in the History Server
-  private[spark] val HISTORY_MAX_APPS =
-    ConfigBuilder("spark.history.maxApplications").intConf.createWithDefault(Integer.MAX_VALUE)
+  // To limit how many applications are shown in the History Server summary ui
+  private[spark] val HISTORY_UI_MAX_APPS =
+    ConfigBuilder("spark.history.ui.maxApplications").intConf.createWithDefault(Integer.MAX_VALUE)
 }

dev/.rat-excludes

@@ -101,4 +101,3 @@ org.apache.spark.scheduler.ExternalClusterManager
 .*\.sql
 .Rbuildignore
 org.apache.spark.deploy.yarn.security.ServiceCredentialProvider
-spark-warehouse

docs/monitoring.md

@@ -118,6 +118,14 @@ The history server can be configured as follows:
       applications will be removed.
     </td>
   </tr>
+  <tr>
+    <td>spark.history.ui.maxApplications</td>
+    <td>Int.MaxValue</td>
+    <td>
+      The number of applications to display on the history summary page. Application UIs are
+      still available even if they are not displayed on the history summary page.
+    </td>
+  </tr>
   <tr>
     <td>spark.history.ui.port</td>
     <td>18080</td>
@@ -242,7 +250,8 @@ can be identified by their `[attempt-id]`. In the API listed below, when running
     <br>Examples:
     <br><code>?minDate=2015-02-10</code>
     <br><code>?minDate=2015-02-03T16:42:40.000GMT</code>
-    <br><code>?maxDate=[date]</code> latest date/time to list; uses same format as <code>minDate</code>.</td>
+    <br><code>?maxDate=[date]</code> latest date/time to list; uses same format as <code>minDate</code>.
+    <br><code>?limit=[limit]</code> limits the number of applications listed.</td>
   </tr>
   <tr>
     <td><code>/applications/[app-id]/jobs</code></td>