Merge pull request #529 from Microsoft/master

* REPL
* improving debug experience
* added new F# example

Author: skaarthik

.gitignore

@@ -46,8 +46,9 @@ SparkCLRCodeCoverage.xml
 AdapterTestCoverage.xml
 TestResult.xml
 csharp/SparkCLR.compiled.nuspec
+cpp/Riosock.sdf
 
 # Office Doc Temp files #
 #########################
-docs/~$SparkCLR.pptx
-docs/~$arkCLR.docx
+docs/~$Mobius.pptx
+docs/~$Mobius.docx

README.md

@@ -25,7 +25,7 @@ metricDataFrame.RegisterTempTable("metrics");
 var joinDataFrame = GetSqlContext().Sql(  
     "SELECT joinedtable.datacenter" +
          ", MAX(joinedtable.latency) maxlatency" +
-         ", AVG(joinedtable.latency) avglatency " + 
+         ", AVG(joinedtable.latency) avglatency " +
     "FROM (" +
        "SELECT a.C1 as datacenter, b.C6 as latency " +  
        "FROM requests a JOIN metrics b ON a.C0  = b.C3) joinedtable " +   
@@ -79,21 +79,21 @@ StreamingContext sparkStreamingContext = StreamingContext.GetOrCreate(checkpoint
 sparkStreamingContext.Start();
 sparkStreamingContext.AwaitTermination();
 ```
-Refer to [Mobius\csharp\Samples](csharp/Samples) directory and [sample usage](csharp/Samples/Microsoft.Spark.CSharp/samplesusage.md) for complete samples.
+Refer to [Mobius\csharp\Samples](./csharp/Samples) directory and [sample usage](./csharp/Samples/Microsoft.Spark.CSharp/samplesusage.md) for complete samples.
 
 ## API Documentation
 
-Refer to [Mobius C# API documentation](csharp/Adapter/documentation/Mobius_API_Documentation.md) for the list of Spark's data processing operations supported in Mobius.
+Refer to [Mobius C# API documentation](./csharp/Adapter/documentation/Mobius_API_Documentation.md) for the list of Spark's data processing operations supported in Mobius.
 
 ## API Usage
 
 Mobius API usage samples are available at:
 
-* [Examples folder](./examples) which contains standalone [C# projects](/notes/running-mobius-app.md#running-mobius-examples-in-local-mode) that can be used as templates to start developing Mobius applications
+* [Examples folder](./examples) which contains standalone [C# projects](./notes/running-mobius-app.md#running-mobius-examples-in-local-mode) that can be used as templates to start developing Mobius applications
 
-* [Samples project](csharp/Samples/Microsoft.Spark.CSharp/) which uses a comprehensive set of Mobius APIs to implement samples that are also used for functional validation of APIs
- 
-* Mobius performance test scenarios implemented in [C#](csharp/Perf/Microsoft.Spark.CSharp) and [Scala](scala/perf) for side by side comparison of Spark driver code
+* [Samples project](./csharp/Samples/Microsoft.Spark.CSharp/) which uses a comprehensive set of Mobius APIs to implement samples that are also used for functional validation of APIs
+
+* Mobius performance test scenarios implemented in [C#](./csharp/Perf/Microsoft.Spark.CSharp) and [Scala](./scala/perf) for side by side comparison of Spark driver code
 
 ## Documents
 
@@ -113,11 +113,12 @@ Refer to the [docs folder](docs) for design overview and other info on Mobius
 |Run samples (functional tests) in local mode |[Samples in Windows](notes/windows-instructions.md#running-samples) |[Samples in Linux](notes/linux-instructions.md#running-mobius-samples-in-linux) |
 |Run examples in local mode |[Examples in Windows](/notes/running-mobius-app.md#running-mobius-examples-in-local-mode) |[Examples in Linux](notes/linux-instructions.md#running-mobius-examples-in-linux) |
 |Run Mobius app |<ul><li>[Standalone cluster](notes/running-mobius-app.md#standalone-cluster)</li><li>[YARN cluster](notes/running-mobius-app.md#yarn-cluster)</li></ul> |<ul><li>[Linux cluster](notes/linux-instructions.md#running-mobius-applications-in-linux)</li><li>[Azure HDInsight Spark Cluster](/notes/linux-instructions.md#mobius-in-azure-hdinsight-spark-cluster)</li><li>[AWS EMR Spark Cluster](/notes/linux-instructions.md#mobius-in-amazon-web-services-emr-spark-cluster)</li> |
+|Run Mobius Shell |<ul><li>[Local](notes/mobius-shell.md#run-shell)</li><li>[YARN](notes/mobius-shell.md#run-shell)</li></ul> | Not supported yet |
 
 ### Useful Links
-* [Configuration parameters in Mobius](/notes/configuration-mobius.md)
-* [Troubleshoot errors in Mobius](/notes/troubleshooting-mobius.md)
-* [Debug Mobius apps](/notes/running-mobius-app.md#debug-mode)
+* [Configuration parameters in Mobius](./notes/configuration-mobius.md)
+* [Troubleshoot errors in Mobius](./notes/troubleshooting-mobius.md)
+* [Debug Mobius apps](./notes/running-mobius-app.md#debug-mode)
 
 ## Supported Spark Versions
 
@@ -129,7 +130,7 @@ Mobius releases are available at https://github.com/Microsoft/Mobius/releases. R
 
 [![NuGet Badge](https://buildstats.info/nuget/Microsoft.SparkCLR)](https://www.nuget.org/packages/Microsoft.SparkCLR)
 
-Refer to [mobius-release-info.md](notes/mobius-release-info.md) for the details on versioning policy and the contents of the release.
+Refer to [mobius-release-info.md](./notes/mobius-release-info.md) for the details on versioning policy and the contents of the release.
 
 ## License
 
@@ -145,7 +146,7 @@ Mobius is licensed under the MIT license. See [LICENSE](LICENSE) file for full l
 [![Join the chat at https://gitter.im/Microsoft/Mobius](https://badges.gitter.im/Microsoft/Mobius.svg)](https://gitter.im/Microsoft/Mobius?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 [![Twitter](https://img.shields.io/twitter/url/http/twitter.com/MobiusForSpark.svg?style=social)](https://twitter.com/intent/tweet?text=@MobiusForSpark [your tweet] via @GitHub)
 
-* Mobius project welcomes contributions. To contribute, follow the instructions in [CONTRIBUTING.md](notes/CONTRIBUTING.md)
+* Mobius project welcomes contributions. To contribute, follow the instructions in [CONTRIBUTING.md](./notes/CONTRIBUTING.md)
 
 * Options to ask your question to the Mobius community
   * create issue on [GitHub](https://github.com/Microsoft/Mobius)

appveyor.yml

@@ -37,13 +37,14 @@ test:
   assemblies:
     - csharp\AdapterTest\bin\Debug\AdapterTest.dll
     - csharp\WorkerTest\bin\Debug\WorkerTest.dll
+    - csharp\ReplTest\bin\Debug\ReplTest.dll
 
 # scripts to run after tests
 after_test:
     - cmd: echo [after_test] ProjectVersion=%ProjectVersion%
-    - .\build\tools\Dynamic-Code-Coverage-Tools\CodeCoverage.exe collect /output:NativeCodeCoverage.coverage .\csharp\packages\NUnit.Console.3.0.1\tools\nunit3-console.exe .\csharp\WorkerTest\bin\Debug\WorkerTest.dll .\csharp\AdapterTest\bin\Debug\AdapterTest.dll
+    - .\build\tools\Dynamic-Code-Coverage-Tools\CodeCoverage.exe collect /output:NativeCodeCoverage.coverage .\csharp\packages\NUnit.Console.3.0.1\tools\nunit3-console.exe .\csharp\WorkerTest\bin\Debug\WorkerTest.dll .\csharp\AdapterTest\bin\Debug\AdapterTest.dll .\csharp\ReplTest\bin\Debug\ReplTest.dll
     - .\build\tools\Dynamic-Code-Coverage-Tools\CodeCoverage.exe analyze /output:NativeCodeCoverage.xml NativeCodeCoverage.coverage
-    - .\csharp\packages\OpenCover.4.6.166\tools\OpenCover.Console.exe -register:user -target:.\csharp\packages\NUnit.Console.3.0.1\tools\nunit3-console.exe -register:user "-targetargs:"".\csharp\WorkerTest\bin\Debug\WorkerTest.dll"" "".\csharp\AdapterTest\bin\Debug\AdapterTest.dll"" " -filter:"+[CSharpWorker*|Microsoft.Spark.CSharp*]*" -excludebyattribute:*.ExcludeFromCodeCoverage* -hideskipped:All -output:.\SparkCLRCodeCoverage.xml
+    - .\csharp\packages\OpenCover.4.6.166\tools\OpenCover.Console.exe -register:user -target:.\csharp\packages\NUnit.Console.3.0.1\tools\nunit3-console.exe -register:user "-targetargs:"".\csharp\WorkerTest\bin\Debug\WorkerTest.dll"" "".\csharp\AdapterTest\bin\Debug\AdapterTest.dll"" "".\csharp\ReplTest\bin\Debug\ReplTest.dll"" " -filter:"+[CSharpWorker*|Microsoft.Spark.CSharp*|Repl]*" -excludebyattribute:*.ExcludeFromCodeCoverage* -hideskipped:All -output:.\SparkCLRCodeCoverage.xml
     - "SET PATH=C:\\Python34;C:\\Python34\\Scripts;%PATH%"
     - pip install codecov
     - codecov -f "NativeCodeCoverage.xml" "SparkCLRCodeCoverage.xml"

build/Build.cmd

@@ -36,6 +36,7 @@ if NOT EXIST "%SPARKCLR_HOME%\bin" mkdir "%SPARKCLR_HOME%\bin"
 if NOT EXIST "%SPARKCLR_HOME%\data" mkdir "%SPARKCLR_HOME%\data"
 if NOT EXIST "%SPARKCLR_HOME%\lib" mkdir "%SPARKCLR_HOME%\lib"
 if NOT EXIST "%SPARKCLR_HOME%\samples" mkdir "%SPARKCLR_HOME%\samples"
+if NOT EXIST "%SPARKCLR_HOME%\repl" mkdir "%SPARKCLR_HOME%\repl"
 
 @echo Assemble Mobius Scala components
 pushd "%CMDHOME%\..\scala"
@@ -153,6 +154,11 @@ copy /y Samples\Microsoft.Spark.CSharp\bin\Release\* "%SPARKCLR_HOME%\samples\"
 
 @echo Mobius Samples data
 copy /y Samples\Microsoft.Spark.CSharp\data\* "%SPARKCLR_HOME%\data\"
+
+@echo SparkCLR REPL
+copy /y Worker\Microsoft.Spark.CSharp\bin\Release\* "%SPARKCLR_HOME%\repl\"
+copy /y Repl\bin\Release\* "%SPARKCLR_HOME%\repl\"
+
 popd
 
 @echo Download external dependencies

csharp/Adapter/Microsoft.Spark.CSharp/Configuration/ConfigurationService.cs

@@ -24,6 +24,7 @@ internal class ConfigurationService : IConfigurationService
         public const string SPARKCLR_HOME = "SPARKCLR_HOME";
         public const string SPARK_MASTER = "spark.master";
         public const string CSHARPBACKEND_PORT = "CSHARPBACKEND_PORT";
+        public const int CSHARPBACKEND_DEBUG_PORT = 5567;
 
         private readonly ILoggerService logger = LoggerServiceFactory.GetLogger(typeof(ConfigurationService));
         private readonly SparkCLRConfiguration configuration;
@@ -178,13 +179,21 @@ internal SparkCLRDebugConfiguration(System.Configuration.Configuration configura
 
             internal override int GetPortNumber()
             {
+                int cSharpBackendPortNumber = CSHARPBACKEND_DEBUG_PORT;
                 KeyValueConfigurationElement portConfig = appSettings.Settings[CSharpBackendPortNumberSettingKey];
                 if (portConfig == null)
                 {
-                    throw new ConfigurationErrorsException(string.Format("Need to set {0} value in App.config for running in DEBUG mode.", CSharpBackendPortNumberSettingKey));
+                    logger.LogInfo(
+                        string.Format(
+                            "Port number not set using setting {0} in App.config. Using default port {1} to connect to CSharpBackend",
+                            CSharpBackendPortNumberSettingKey, cSharpBackendPortNumber));
                 }
-                int cSharpBackendPortNumber = int.Parse(portConfig.Value);
-                logger.LogInfo(string.Format("CSharpBackend port number read from app config {0}", cSharpBackendPortNumber));
+                else
+                {
+                    cSharpBackendPortNumber = int.Parse(portConfig.Value);
+                    logger.LogInfo(string.Format("CSharpBackend port number read from app config {0}. Using it to connect to CSharpBackend", cSharpBackendPortNumber));
+                }
+                
                 return cSharpBackendPortNumber;
             }
 

csharp/Adapter/Microsoft.Spark.CSharp/Core/SparkContext.cs

@@ -32,12 +32,20 @@ public class SparkContext
         /// </summary>
         internal static SparkContext GetActiveSparkContext()
         {
-                return _activeSparkContext;
+            return _activeSparkContext;
         }
 
         private AccumulatorServer accumulatorServer;
         private int nextAccumulatorId;
 
+        /// <summary>
+        /// Return a copy of this JavaSparkContext's configuration. The configuration ''cannot'' be changed at runtime.
+        /// </summary>
+        public SparkConf GetConf()
+        {
+            return new SparkConf(SparkContextProxy.GetConf());
+        }
+
         /// <summary>
         /// The version of Spark on which this application is running.
         /// </summary>
@@ -88,7 +96,7 @@ public int DefaultMinPartitions
         /// <param name="sparkHome">the path that holds spark bits</param>
         public SparkContext(string master, string appName, string sparkHome)
             : this(master, appName, sparkHome, null)
-        {}
+        { }
 
         /// <summary>
         /// Initializes a SparkContext instance with a specific master and application name.
@@ -97,15 +105,15 @@ public SparkContext(string master, string appName, string sparkHome)
         /// <param name="appName"></param>
         public SparkContext(string master, string appName)
             : this(master, appName, null, null)
-        {}
+        { }
 
         /// <summary>
         /// Initializes a SparkContext instance with a specific spark config.
         /// </summary>
         /// <param name="conf">A SparkConf object that represents the settings for spark</param>
         public SparkContext(SparkConf conf)
             : this(null, null, null, conf)
-        {}
+        { }
 
         /// <summary>
         /// when created from checkpoint
@@ -584,6 +592,27 @@ internal static byte[] BuildCommand(CSharpWorkerFunc workerFunc, SerializedMode
             Array.Reverse(lengthAsBytes);
             commandPayloadBytesList.Add(lengthAsBytes);
             commandPayloadBytesList.Add(modeBytes);
+
+            // add run mode
+            // N - normal
+            // R - repl
+            var runMode = Environment.GetEnvironmentVariable("SPARKCLR_RUN_MODE") ?? "N";
+            var runModeBytes = Encoding.UTF8.GetBytes(runMode);
+            lengthAsBytes = BitConverter.GetBytes(runModeBytes.Length);
+            Array.Reverse(lengthAsBytes);
+            commandPayloadBytesList.Add(lengthAsBytes);
+            commandPayloadBytesList.Add(runModeBytes);
+
+            if ("R".Equals(runMode, StringComparison.InvariantCultureIgnoreCase))
+            {
+                // add compilation dump directory
+                var compilationDumpDirBytes = Encoding.UTF8.GetBytes(Environment.GetEnvironmentVariable("SPARKCLR_SCRIPT_COMPILATION_DIR")?? ".");
+                lengthAsBytes = BitConverter.GetBytes(compilationDumpDirBytes.Length);
+                Array.Reverse(lengthAsBytes);
+                commandPayloadBytesList.Add(lengthAsBytes);
+                commandPayloadBytesList.Add(compilationDumpDirBytes);
+            }
+
             // add func
             var funcBytes = stream.ToArray();
             var funcBytesLengthAsBytes = BitConverter.GetBytes(funcBytes.Length);

csharp/Adapter/Microsoft.Spark.CSharp/Interop/SparkCLREnvironment.cs

@@ -11,6 +11,7 @@
 [assembly: InternalsVisibleTo("Tests.Common")]
 [assembly: InternalsVisibleTo("AdapterTest")]
 [assembly: InternalsVisibleTo("WorkerTest")]
+[assembly: InternalsVisibleTo("ReplTest")] 
 // DynamicProxyGenAssembly2 is a temporary assembly built by mocking systems that use CastleProxy like Moq
 [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")]
 namespace Microsoft.Spark.CSharp.Interop

csharp/Adapter/Microsoft.Spark.CSharp/Proxy/ISparkContextProxy.cs

@@ -14,6 +14,7 @@ namespace Microsoft.Spark.CSharp.Proxy
 {
     internal interface ISparkContextProxy
     {
+        ISparkConfProxy GetConf();
         ISqlContextProxy CreateSqlContext();
         ISqlContextProxy CreateHiveContext();
         IColumnProxy CreateColumnFromName(string name);

csharp/Adapter/Microsoft.Spark.CSharp/Proxy/Ipc/SparkContextIpcProxy.cs

@@ -23,6 +23,11 @@ internal class SparkContextIpcProxy : ISparkContextProxy
         private JvmObjectReference jvmAccumulatorReference;
         internal List<JvmObjectReference> jvmBroadcastReferences = new List<JvmObjectReference>();
 
+        public ISparkConfProxy GetConf()
+        {
+            return new SparkConfIpcProxy(new JvmObjectReference((string)SparkCLRIpcProxy.JvmBridge.CallNonStaticJavaMethod(jvmJavaContextReference, "getConf")));
+        }  
+
         internal JvmObjectReference JvmSparkContextReference
         {
             get { return jvmSparkContextReference; }

csharp/AdapterTest/ConfigurationServiceTest.cs

@@ -39,7 +39,7 @@ public void TestDebugConfiguration()
             try
             {
                 ConfigurationService debugConfiguration = new ConfigurationService();
-                Assert.Throws<ConfigurationErrorsException>(() => Console.WriteLine(debugConfiguration.BackendPortNumber));
+                Assert.AreEqual(ConfigurationService.CSHARPBACKEND_DEBUG_PORT, debugConfiguration.BackendPortNumber);
                 Assert.Throws<Exception>(() => Console.WriteLine(debugConfiguration.GetCSharpWorkerExePath()));
             }
             finally