change options of spark mirror url

Author: junyangq

R/pkg/DESCRIPTION

@@ -8,8 +8,6 @@ Maintainer: Shivaram Venkataraman <shivaram@cs.berkeley.edu>
 Depends:
     R (>= 3.0),
     methods
-Imports:
-    rappdirs
 Suggests:
     testthat,
     e1071,

R/pkg/R/install.R

@@ -18,7 +18,7 @@
 # Functions to install Spark in case the user directly downloads SparkR
 # from CRAN.
 
-#' Download and Install Spark to Local Directory
+#' Download and Install Spark Core to Local Directory
 #' 
 #' \code{install_spark} downloads and installs Spark to local directory if
 #' it is not found. The Spark version we use is 2.0.0 (preview). Users can
@@ -27,7 +27,7 @@
 #'
 #' @param hadoop_version Version of Hadoop to install. 2.3, 2.4, 2.6,
 #'        and 2.7 (default)
-#' @param url the base URL of the repositories to use
+#' @param mirror_url the base URL of the repositories to use
 #' @param local_dir local directory that Spark is installed to
 #' @return \code{install_spark} returns the local directory 
 #'         where Spark is found or installed
@@ -39,47 +39,118 @@
 #' install_spark()
 #'}
 #' @note install_spark since 2.1.0
-install_spark <- function(hadoop_version = NULL, url = NULL, local_dir = NULL) {
+install_spark <- function(hadoop_version = NULL, mirror_url = NULL,
+                          local_dir = NULL) {
   version <- paste0("spark-", spark_version_default())
-  hadoop_version <- hadoop_version_default()
-  packageName <- paste0(version, "-bin-hadoop", hadoop_version)
+  hadoop_version <- match.arg(hadoop_version, supported_versions_hadoop())
+  packageName <- ifelse(hadoop_version == "without",
+                        paste0(version, "-bin-without-hadoop"),
+                        paste0(version, "-bin-hadoop", hadoop_version))
   if (is.null(local_dir)) {
-    local_dir <- getOption("spark.install.dir",
-                           rappdirs::app_dir("spark"))$cache()
+    local_dir <- getOption("spark.install.dir",spark_cache_path())
+  } else {
+    local_dir <- normalizePath(local_dir)
   }
+
   packageLocalDir <- file.path(local_dir, packageName)
+
   if (dir.exists(packageLocalDir)) {
     fmt <- "Spark %s for Hadoop %s has been installed."
     msg <- sprintf(fmt, version, hadoop_version)
     message(msg)
+    return(invisible(packageLocalDir))
+  }
+
+  packageLocalPath <- paste0(packageLocalDir, ".tgz")
+  tarExists <- file.exists(packageLocalPath)
+
+  if (tarExists) {
+    message("Tar file found. Installing...")
   } else {
     dir.create(packageLocalDir, recursive = TRUE)
-    if (is.null(url)) {
-      mirror_sites <- read.csv(mirror_csv_url())
-      url <- mirror_sites$url[1]
+    if (is.null(mirror_url)) {
+      message("Remote URL not provided. Use Apache default.")
+      mirror_url <- mirror_url_default()
     }
-    packageRemotePath <- paste0(file.path(url, "spark", version, packageName),
+    # This is temporary, should be removed when released
+    version <- "spark-releases/spark-2.0.0-rc4-bin"
+    packageRemotePath <- paste0(file.path(mirror_url, version, packageName),
                                 ".tgz")
     fmt <- paste("Installing Spark %s for Hadoop %s.",
                  "Downloading from:\n %s",
                  "Installing to:\n %s", sep = "\n")
     msg <- sprintf(fmt, version, hadoop_version, packageRemotePath,
                    packageLocalDir)
     message(msg)
-    packageLocalPath <- paste0(packageLocalDir, ".tgz")
-    download.file(packageRemotePath, packageLocalPath)
-    untar(tarfile = packageLocalPath, exdir = local_dir)
+
+    fetchFail <- tryCatch(download.file(packageRemotePath, packageLocalPath),
+                          error = function(e) {
+                            msg <- paste0("Fetch failed from ", mirror_url, ".")
+                            message(msg)
+                            TRUE
+                          })
+    if (fetchFail) {
+      message("Try the backup option.")
+      mirror_sites <- tryCatch(read.csv(mirror_url_csv()),
+                               error = function(e) stop("No csv file found."))
+      mirror_url <- mirror_sites$url[1]
+      packageRemotePath <- paste0(file.path(mirror_url, version, packageName),
+                                  ".tgz")
+      message(sprintf("Downloading from:\n %s", packageRemotePath))
+      tryCatch(download.file(packageRemotePath, packageLocalPath),
+               error = function(e) {
+                 stop("Download failed. Please provide a valid mirror_url.")
+               })
+    }
+  }
+
+  untar(tarfile = packageLocalPath, exdir = local_dir)
+  if (!tarExists) {
     unlink(packageLocalPath)
   }
-  packageLocalDir
+  message("Installation done.")
+  invisible(packageLocalDir)
+}
+
+mirror_url_default <- function() {
+  # change to http://www.apache.org/dyn/closer.lua
+  # when released
+
+  "http://people.apache.org/~pwendell"
+}
+
+supported_versions_hadoop <- function() {
+  c("2.7", "2.6", "2.4", "without")
+}
+
+spark_cache_path <- function() {
+  if (.Platform$OS.type == "windows") {
+    winAppPath <- Sys.getenv("%LOCALAPPDATA%", unset = NA)
+    if (is.null(winAppPath)) {
+      msg <- paste("%LOCALAPPDATA% not found.",
+                    "Please define or enter an installation path in loc_dir.")
+      stop(msg)
+    } else {
+      path <- file.path(winAppPath, "spark", "spark", "Cache")
+    }
+  } else if (.Platform$OS.type == "unix") {
+    if (Sys.info()["sysname"] == "Darwin") {
+      path <- file.path("~/Library/Caches", "spark")
+    } else {
+      path <- file.path(Sys.getenv("XDG_CACHE_HOME", "~/.cache"), "spark")
+    }
+  } else {
+    stop("Unknown OS")
+  }
+  normalizePath(path, mustWork = TRUE)
 }
 
-mirror_csv_url <- function() {
+mirror_url_csv <- function() {
   system.file("extdata", "spark_download.csv", package = "SparkR")
 }
 
 spark_version_default <- function() {
-  "2.0.0-preview"
+  "2.0.0"
 }
 
 hadoop_version_default <- function() {

R/pkg/R/sparkR.R

@@ -366,8 +366,12 @@ sparkR.session <- function(
     overrideEnvs(sparkConfigMap, paramMap)
   }
   if (!nzchar(master) || master_is_local(master)) {
-    if (!nzchar(sparkHome) || !dir.exists(sparkHome)) {
-      message("Spark is not found in SPARK_HOME. Redirect to the cache directory.")
+    if (!dir.exists(sparkHome)) {
+      fmt <- paste("Spark not found in SPARK_HOME: %s.\n",
+                   "Search in the cache directory.",
+                   "It will be installed if not found.")
+      msg <- sprintf(fmt, sparkHome)
+      message(msg)
       packageLocalDir <- install_spark()
       sparkHome <- packageLocalDir
     } else {
@@ -376,7 +380,7 @@ sparkR.session <- function(
       message(msg)
     }
   }
-  
+
   if (!exists(".sparkRjsc", envir = .sparkREnv)) {
     sparkExecutorEnvMap <- new.env()
     sparkR.sparkContext(master, appName, sparkHome, sparkConfigMap, sparkExecutorEnvMap,