SHS-NG M4.5: Simplify API resource structure.

With the new UI store, the API resource classes have a lot less code,
since there's no need for complicated translations between the UI
types and the API types. So the code ended up with a bunch of files
with a single method declared in them.

This change re-structures the API code so that it uses less classes;
mainly, most sub-resources were removed, and the code to deal with
single-attempt and multi-attempt apps was simplified.

The only change was the addition of a method to return a single
attempt's information; that was missing in the old API, so trying
to retrieve "/v1/applications/appId/attemptId" would result in a
404 even if the attempt existed (and URIs under that one would
return valid data).

The streaming API resources also overtook the same treatment; the
streaming backend is still not hooked up to the store, but once it
is, the code in the remaining classes will be simplified even
further.

Author: Marcelo Vanzin

common/kvstore/src/main/java/org/apache/spark/kvstore/LevelDB.java

@@ -177,6 +177,8 @@ public void delete(Class<?> type, Object naturalKey, boolean sync) throws Except
           batch.write(sync);
         }
       }
+    } catch (NoSuchElementException nse) {
+      // Ignore.
     } finally {
       batch.close();
     }

core/src/main/scala/org/apache/spark/status/api/v1/AllExecutorListResource.scala

@@ -44,189 +44,14 @@ import org.apache.spark.ui.SparkUI
 private[v1] class ApiRootResource extends ApiRequestContext {
 
   @Path("applications")
-  def getApplicationList(): ApplicationListResource = {
-    new ApplicationListResource(uiRoot)
-  }
+  def applicationList(): Class[ApplicationListResource] = classOf[ApplicationListResource]
 
   @Path("applications/{appId}")
-  def getApplication(): OneApplicationResource = {
-    new OneApplicationResource(uiRoot)
-  }
-
-  @Path("applications/{appId}/{attemptId}/jobs")
-  def getJobs(
-      @PathParam("appId") appId: String,
-      @PathParam("attemptId") attemptId: String): AllJobsResource = {
-    withSparkUI(appId, Some(attemptId)) { ui =>
-      new AllJobsResource(ui)
-    }
-  }
-
-  @Path("applications/{appId}/jobs")
-  def getJobs(@PathParam("appId") appId: String): AllJobsResource = {
-    withSparkUI(appId, None) { ui =>
-      new AllJobsResource(ui)
-    }
-  }
-
-  @Path("applications/{appId}/jobs/{jobId: \\d+}")
-  def getJob(@PathParam("appId") appId: String): OneJobResource = {
-    withSparkUI(appId, None) { ui =>
-      new OneJobResource(ui)
-    }
-  }
-
-  @Path("applications/{appId}/{attemptId}/jobs/{jobId: \\d+}")
-  def getJob(
-      @PathParam("appId") appId: String,
-      @PathParam("attemptId") attemptId: String): OneJobResource = {
-    withSparkUI(appId, Some(attemptId)) { ui =>
-      new OneJobResource(ui)
-    }
-  }
-
-  @Path("applications/{appId}/executors")
-  def getExecutors(@PathParam("appId") appId: String): ExecutorListResource = {
-    withSparkUI(appId, None) { ui =>
-      new ExecutorListResource(ui)
-    }
-  }
-
-  @Path("applications/{appId}/allexecutors")
-  def getAllExecutors(@PathParam("appId") appId: String): AllExecutorListResource = {
-    withSparkUI(appId, None) { ui =>
-      new AllExecutorListResource(ui)
-    }
-  }
-
-  @Path("applications/{appId}/{attemptId}/executors")
-  def getExecutors(
-      @PathParam("appId") appId: String,
-      @PathParam("attemptId") attemptId: String): ExecutorListResource = {
-    withSparkUI(appId, Some(attemptId)) { ui =>
-      new ExecutorListResource(ui)
-    }
-  }
-
-  @Path("applications/{appId}/{attemptId}/allexecutors")
-  def getAllExecutors(
-      @PathParam("appId") appId: String,
-      @PathParam("attemptId") attemptId: String): AllExecutorListResource = {
-    withSparkUI(appId, Some(attemptId)) { ui =>
-      new AllExecutorListResource(ui)
-    }
-  }
-
-  @Path("applications/{appId}/stages")
-  def getStages(@PathParam("appId") appId: String): AllStagesResource = {
-    withSparkUI(appId, None) { ui =>
-      new AllStagesResource(ui)
-    }
-  }
-
-  @Path("applications/{appId}/{attemptId}/stages")
-  def getStages(
-      @PathParam("appId") appId: String,
-      @PathParam("attemptId") attemptId: String): AllStagesResource = {
-    withSparkUI(appId, Some(attemptId)) { ui =>
-      new AllStagesResource(ui)
-    }
-  }
-
-  @Path("applications/{appId}/stages/{stageId: \\d+}")
-  def getStage(@PathParam("appId") appId: String): OneStageResource = {
-    withSparkUI(appId, None) { ui =>
-      new OneStageResource(ui)
-    }
-  }
-
-  @Path("applications/{appId}/{attemptId}/stages/{stageId: \\d+}")
-  def getStage(
-      @PathParam("appId") appId: String,
-      @PathParam("attemptId") attemptId: String): OneStageResource = {
-    withSparkUI(appId, Some(attemptId)) { ui =>
-      new OneStageResource(ui)
-    }
-  }
-
-  @Path("applications/{appId}/storage/rdd")
-  def getRdds(@PathParam("appId") appId: String): AllRDDResource = {
-    withSparkUI(appId, None) { ui =>
-      new AllRDDResource(ui)
-    }
-  }
-
-  @Path("applications/{appId}/{attemptId}/storage/rdd")
-  def getRdds(
-      @PathParam("appId") appId: String,
-      @PathParam("attemptId") attemptId: String): AllRDDResource = {
-    withSparkUI(appId, Some(attemptId)) { ui =>
-      new AllRDDResource(ui)
-    }
-  }
-
-  @Path("applications/{appId}/storage/rdd/{rddId: \\d+}")
-  def getRdd(@PathParam("appId") appId: String): OneRDDResource = {
-    withSparkUI(appId, None) { ui =>
-      new OneRDDResource(ui)
-    }
-  }
-
-  @Path("applications/{appId}/{attemptId}/storage/rdd/{rddId: \\d+}")
-  def getRdd(
-      @PathParam("appId") appId: String,
-      @PathParam("attemptId") attemptId: String): OneRDDResource = {
-    withSparkUI(appId, Some(attemptId)) { ui =>
-      new OneRDDResource(ui)
-    }
-  }
-
-  @Path("applications/{appId}/logs")
-  def getEventLogs(
-      @PathParam("appId") appId: String): EventLogDownloadResource = {
-    try {
-      // withSparkUI will throw NotFoundException if attemptId exists for this application.
-      // So we need to try again with attempt id "1".
-      withSparkUI(appId, None) { _ =>
-        new EventLogDownloadResource(uiRoot, appId, None)
-      }
-    } catch {
-      case _: NotFoundException =>
-        withSparkUI(appId, Some("1")) { _ =>
-          new EventLogDownloadResource(uiRoot, appId, None)
-        }
-    }
-  }
-
-  @Path("applications/{appId}/{attemptId}/logs")
-  def getEventLogs(
-      @PathParam("appId") appId: String,
-      @PathParam("attemptId") attemptId: String): EventLogDownloadResource = {
-    withSparkUI(appId, Some(attemptId)) { _ =>
-      new EventLogDownloadResource(uiRoot, appId, Some(attemptId))
-    }
-  }
+  def application(): Class[OneApplicationResource] = classOf[OneApplicationResource]
 
   @Path("version")
-  def getVersion(): VersionResource = {
-    new VersionResource(uiRoot)
-  }
-
-  @Path("applications/{appId}/environment")
-  def getEnvironment(@PathParam("appId") appId: String): ApplicationEnvironmentResource = {
-    withSparkUI(appId, None) { ui =>
-      new ApplicationEnvironmentResource(ui)
-    }
-  }
+  def version(): VersionInfo = new VersionInfo(org.apache.spark.SPARK_VERSION)
 
-  @Path("applications/{appId}/{attemptId}/environment")
-  def getEnvironment(
-      @PathParam("appId") appId: String,
-      @PathParam("attemptId") attemptId: String): ApplicationEnvironmentResource = {
-    withSparkUI(appId, Some(attemptId)) { ui =>
-      new ApplicationEnvironmentResource(ui)
-    }
-  }
 }
 
 private[spark] object ApiRootResource {
@@ -293,23 +118,29 @@ private[v1] trait ApiRequestContext {
 
   def uiRoot: UIRoot = UIRootFromServletContext.getUiRoot(servletContext)
 
+}
 
-  /**
-   * Get the spark UI with the given appID, and apply a function
-   * to it.  If there is no such app, throw an appropriate exception
-   */
-  def withSparkUI[T](appId: String, attemptId: Option[String])(f: SparkUI => T): T = {
+/**
+ * Base class for resource handlers that use app-specific data. Abstracts away dealing with
+ * application and attempt IDs, and finding the app's UI.
+ */
+private[v1] trait BaseAppResource extends ApiRequestContext {
+
+  @PathParam("appId") protected[this] var appId: String = _
+  @PathParam("attemptId") protected[this] var attemptId: String = _
+
+  protected def withUI[T](fn: SparkUI => T): T = {
     try {
-      uiRoot.withSparkUI(appId, attemptId) { ui =>
+      uiRoot.withSparkUI(appId, Option(attemptId)) { ui =>
         val user = httpRequest.getRemoteUser()
         if (!ui.securityManager.checkUIViewPermissions(user)) {
           throw new ForbiddenException(raw"""user "$user" is not authorized""")
         }
-        f(ui)
+        fn(ui)
       }
     } catch {
       case _: NoSuchElementException =>
-        val appKey = attemptId.map(appId + "/" + _).getOrElse(appId)
+        val appKey = Option(attemptId).map(appId + "/" + _).getOrElse(appId)
         throw new NotFoundException(s"no such app: $appKey")
     }
   }