Changes of the examples after API improvements

Author: Yunni

docs/ml-features.md

@@ -1514,15 +1514,15 @@ Bucketed Random Projection accepts arbitrary vectors as input features, and supp
 Refer to the [RandomProjection Scala docs](api/scala/index.html#org.apache.spark.ml.feature.RandomProjection)
 for more details on the API.
 
-{% include_example scala/org/apache/spark/examples/ml/RandomProjectionExample.scala %}
+{% include_example scala/org/apache/spark/examples/ml/BucketedRandomProjectionLSHExample.scala %}
 </div>
 
 <div data-lang="java" markdown="1">
 
 Refer to the [RandomProjection Java docs](api/java/org/apache/spark/ml/feature/RandomProjection.html)
 for more details on the API.
 
-{% include_example java/org/apache/spark/examples/ml/JavaRandomProjectionExample.java %}
+{% include_example java/org/apache/spark/examples/ml/JavaBucketedRandomProjectionLSHExample.java %}
 </div>
 </div>
 
@@ -1546,15 +1546,15 @@ The input sets for MinHash are represented as binary vectors, where the vector i
 Refer to the [MinHash Scala docs](api/scala/index.html#org.apache.spark.ml.feature.MinHash)
 for more details on the API.
 
-{% include_example scala/org/apache/spark/examples/ml/MinHashExample.scala %}
+{% include_example scala/org/apache/spark/examples/ml/MinHashLSHExample.scala %}
 </div>
 
 <div data-lang="java" markdown="1">
 
 Refer to the [MinHash Java docs](api/java/org/apache/spark/ml/feature/MinHash.html)
 for more details on the API.
 
-{% include_example java/org/apache/spark/examples/ml/JavaMinHashExample.java %}
+{% include_example java/org/apache/spark/examples/ml/JavaMinHashLSHExample.java %}
 </div>
 </div>
 

examples/src/main/java/org/apache/spark/examples/ml/JavaApproxNearestNeighborExample.java

@@ -23,8 +23,8 @@
 import java.util.Arrays;
 import java.util.List;
 
-import org.apache.spark.ml.feature.MinHash;
-import org.apache.spark.ml.feature.MinHashModel;
+import org.apache.spark.ml.feature.MinHashLSH;
+import org.apache.spark.ml.feature.MinHashLSHModel;
 import org.apache.spark.ml.linalg.Vector;
 import org.apache.spark.ml.linalg.VectorUDT;
 import org.apache.spark.ml.linalg.Vectors;
@@ -57,27 +57,15 @@ public static void main(String[] args) {
     });
     Dataset<Row> dataFrame = spark.createDataFrame(data, schema);
 
-    MinHash mh = new MinHash()
-      .setOutputDim(5)
+    MinHashLSH mh = new MinHashLSH()
+      .setNumHashTables(5)
       .setInputCol("keys")
       .setOutputCol("values");
 
     Vector key1 = Vectors.sparse(6, new int[]{1, 3}, new double[]{1.0, 1.0, 1.0});
-    Vector key2 = Vectors.sparse(6, new int[]{5}, new double[]{1.0, 1.0, 1.0});
 
-    MinHashModel model = mh.fit(dataFrame);
+    MinHashLSHModel model = mh.fit(dataFrame);
     model.approxNearestNeighbors(dataFrame, key1, 2).show();
-
-    System.out.println("Difference between single probing and multi probing:");
-
-    System.out.println("Single probing sometimes returns less than k rows");
-    model.approxNearestNeighbors(dataFrame, key2, 3, true, "distCol").show();
-
-    System.out.println("Multi probing returns exact k rows whenever possible");
-    model.approxNearestNeighbors(dataFrame, key2, 3, false, "distCol").show();
-
-    System.out.println("Multi probing returns the whole dataset when there are not enough rows");
-    model.approxNearestNeighbors(dataFrame, key2, 4, false, "distCol").show();
     // $example off$
 
     spark.stop();

examples/src/main/java/org/apache/spark/examples/ml/JavaApproxSimilarityJoinExample.java

@@ -23,8 +23,8 @@
 import java.util.Arrays;
 import java.util.List;
 
-import org.apache.spark.ml.feature.MinHash;
-import org.apache.spark.ml.feature.MinHashModel;
+import org.apache.spark.ml.feature.MinHashLSH;
+import org.apache.spark.ml.feature.MinHashLSHModel;
 import org.apache.spark.ml.linalg.VectorUDT;
 import org.apache.spark.ml.linalg.Vectors;
 import org.apache.spark.sql.Dataset;
@@ -63,12 +63,12 @@ public static void main(String[] args) {
     Dataset<Row> dfA = spark.createDataFrame(dataA, schema);
     Dataset<Row> dfB = spark.createDataFrame(dataB, schema);
 
-    MinHash mh = new MinHash()
-      .setOutputDim(5)
+    MinHashLSH mh = new MinHashLSH()
+      .setNumHashTables(5)
       .setInputCol("keys")
       .setOutputCol("values");
 
-    MinHashModel model = mh.fit(dfA);
+    MinHashLSHModel model = mh.fit(dfA);
     model.approxSimilarityJoin(dfA, dfB, 0.6).show();
 
     // Cache the transformed columns

…ples/ml/JavaRandomProjectionExample.java …aBucketedRandomProjectionLSHExample.javaexamples/src/main/java/org/apache/spark/examples/ml/JavaRandomProjectionExample.java renamed to examples/src/main/java/org/apache/spark/examples/ml/JavaBucketedRandomProjectionLSHExample.java examples/src/main/java/org/apache/spark/examples/ml/JavaRandomProjectionExample.java renamed to examples/src/main/java/org/apache/spark/examples/ml/JavaBucketedRandomProjectionLSHExample.java

@@ -23,8 +23,8 @@
 import java.util.Arrays;
 import java.util.List;
 
-import org.apache.spark.ml.feature.RandomProjection;
-import org.apache.spark.ml.feature.RandomProjectionModel;
+import org.apache.spark.ml.feature.BucketedRandomProjectionLSH;
+import org.apache.spark.ml.feature.BucketedRandomProjectionLSHModel;
 import org.apache.spark.ml.linalg.VectorUDT;
 import org.apache.spark.ml.linalg.Vectors;
 import org.apache.spark.sql.Dataset;
@@ -36,11 +36,11 @@
 import org.apache.spark.sql.types.StructType;
 // $example off$
 
-public class JavaRandomProjectionExample {
+public class JavaBucketedRandomProjectionLSHExample {
   public static void main(String[] args) {
     SparkSession spark = SparkSession
       .builder()
-      .appName("JavaRandomProjectionExample")
+      .appName("JavaBucketedRandomProjectionLSHExample")
       .getOrCreate();
 
     // $example on$
@@ -57,13 +57,13 @@ public static void main(String[] args) {
     });
     Dataset<Row> dataFrame = spark.createDataFrame(data, schema);
 
-    RandomProjection mh = new RandomProjection()
+    BucketedRandomProjectionLSH mh = new BucketedRandomProjectionLSH()
       .setBucketLength(2.0)
-      .setOutputDim(1)
+      .setNumHashTables(1)
       .setInputCol("keys")
       .setOutputCol("values");
 
-    RandomProjectionModel model = mh.fit(dataFrame);
+    BucketedRandomProjectionLSHModel model = mh.fit(dataFrame);
     model.transform(dataFrame).show();
     // $example off$
 

examples/src/main/java/org/apache/spark/examples/ml/JavaLSHTransformationExample.java

@@ -23,8 +23,8 @@
 import java.util.Arrays;
 import java.util.List;
 
-import org.apache.spark.ml.feature.MinHash;
-import org.apache.spark.ml.feature.MinHashModel;
+import org.apache.spark.ml.feature.MinHashLSH;
+import org.apache.spark.ml.feature.MinHashLSHModel;
 import org.apache.spark.ml.linalg.VectorUDT;
 import org.apache.spark.ml.linalg.Vectors;
 import org.apache.spark.sql.Dataset;
@@ -57,20 +57,19 @@ public static void main(String[] args) {
         Dataset<Row> dataFrame = spark.createDataFrame(data, schema);
 
         // Single LSH hashing
-        MinHash mhSingleHash = new MinHash()
-                .setOutputDim(1)
+        MinHashLSH mhSingleHash = new MinHashLSH()
                 .setInputCol("keys")
                 .setOutputCol("values");
-        MinHashModel modelSingleHash = mhSingleHash.fit(dataFrame);
+        MinHashLSHModel modelSingleHash = mhSingleHash.fit(dataFrame);
         // Feature transformation: add a new column for a hash value
         modelSingleHash.transform(dataFrame).show();
 
         // Use more than 1 hash functions
-        MinHash mh = new MinHash()
-                .setOutputDim(5)
+        MinHashLSH mh = new MinHashLSH()
+                .setNumHashTables(5)
                 .setInputCol("keys")
                 .setOutputCol("values");
-        MinHashModel model = mh.fit(dataFrame);
+        MinHashLSHModel model = mh.fit(dataFrame);
         // Feature Transformation: add a new column for multiple hash values
         model.transform(dataFrame).show();
         // $example off$

…park/examples/ml/JavaMinHashExample.java …k/examples/ml/JavaMinHashLSHExample.javaexamples/src/main/java/org/apache/spark/examples/ml/JavaMinHashExample.java renamed to examples/src/main/java/org/apache/spark/examples/ml/JavaMinHashLSHExample.java examples/src/main/java/org/apache/spark/examples/ml/JavaMinHashExample.java renamed to examples/src/main/java/org/apache/spark/examples/ml/JavaMinHashLSHExample.java

@@ -1,11 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.examples.ml;
+
 import org.apache.spark.sql.SparkSession;
 
 // $example on$
 import java.util.Arrays;
 import java.util.List;
 
-import org.apache.spark.ml.feature.MinHash;
-import org.apache.spark.ml.feature.MinHashModel;
+import org.apache.spark.ml.feature.MinHashLSH;
+import org.apache.spark.ml.feature.MinHashLSHModel;
 import org.apache.spark.ml.linalg.VectorUDT;
 import org.apache.spark.ml.linalg.Vectors;
 import org.apache.spark.sql.Dataset;
@@ -17,11 +36,11 @@
 import org.apache.spark.sql.types.StructType;
 // $example off$
 
-public class JavaMinHashExample {
+public class JavaMinHashLSHExample {
   public static void main(String[] args) {
     SparkSession spark = SparkSession
       .builder()
-      .appName("JavaMinHashExample")
+      .appName("JavaMinHashLSHExample")
       .getOrCreate();
 
     // $example on$
@@ -37,12 +56,12 @@ public static void main(String[] args) {
     });
     Dataset<Row> dataFrame = spark.createDataFrame(data, schema);
 
-    MinHash mh = new MinHash()
-      .setOutputDim(1)
+    MinHashLSH mh = new MinHashLSH()
+      .setNumHashTables(1)
       .setInputCol("keys")
       .setOutputCol("values");
 
-    MinHashModel model = mh.fit(dataFrame);
+    MinHashLSHModel model = mh.fit(dataFrame);
     model.transform(dataFrame).show();
     // $example off$
 

examples/src/main/scala/org/apache/spark/examples/ml/ApproxNearestNeighborExample.scala

@@ -19,7 +19,7 @@
 package org.apache.spark.examples.ml
 
 // $example on$
-import org.apache.spark.ml.feature.MinHash
+import org.apache.spark.ml.feature.MinHashLSH
 import org.apache.spark.ml.linalg.Vectors
 // $example off$
 import org.apache.spark.sql.SparkSession
@@ -39,27 +39,15 @@ object ApproxNearestNeighborExample {
       (2, Vectors.sparse(6, Seq((0, 1.0), (2, 1.0), (4, 1.0))))
     )).toDF("id", "keys")
 
-    val mh = new MinHash()
-      .setOutputDim(5)
+    val mh = new MinHashLSH()
+      .setNumHashTables(5)
       .setInputCol("keys")
       .setOutputCol("values")
 
     val key1 = Vectors.sparse(6, Seq((1, 1.0), (3, 1.0)))
-    val key2 = Vectors.sparse(6, Seq((5, 1.0)))
 
     val model = mh.fit(dataFrame)
     model.approxNearestNeighbors(dataFrame, key1, 2).show()
-
-    println("Difference between single probing and multi probing:")
-
-    println("single probing sometimes returns less than k rows")
-    model.approxNearestNeighbors(dataFrame, key2, 3, true, "distCol").show()
-
-    println("multi probing returns exact k rows whenever possible")
-    model.approxNearestNeighbors(dataFrame, key2, 3, false, "distCol").show()
-
-    println("multi probing returns the whole dataset when there are not enough rows")
-    model.approxNearestNeighbors(dataFrame, key2, 4, false, "distCol").show()
     // $example off$
 
     spark.stop()

examples/src/main/scala/org/apache/spark/examples/ml/ApproxSimilarityJoinExample.scala

@@ -19,7 +19,7 @@
 package org.apache.spark.examples.ml
 
 // $example on$
-import org.apache.spark.ml.feature.MinHash
+import org.apache.spark.ml.feature.MinHashLSH
 import org.apache.spark.ml.linalg.Vectors
 // $example off$
 import org.apache.spark.sql.SparkSession
@@ -45,8 +45,8 @@ object ApproxSimilarityJoinExample {
       (5, Vectors.sparse(6, Seq((1, 1.0), (2, 1.0), (4, 1.0))))
     )).toDF("id", "keys")
 
-    val mh = new MinHash()
-      .setOutputDim(5)
+    val mh = new MinHashLSH()
+      .setNumHashTables(5)
       .setInputCol("keys")
       .setOutputCol("values")
 

…xamples/ml/RandomProjectionExample.scala …BucketedRandomProjectionLSHExample.scalaexamples/src/main/scala/org/apache/spark/examples/ml/RandomProjectionExample.scala renamed to examples/src/main/scala/org/apache/spark/examples/ml/BucketedRandomProjectionLSHExample.scala examples/src/main/scala/org/apache/spark/examples/ml/RandomProjectionExample.scala renamed to examples/src/main/scala/org/apache/spark/examples/ml/BucketedRandomProjectionLSHExample.scala

@@ -19,17 +19,17 @@
 package org.apache.spark.examples.ml
 
 // $example on$
-import org.apache.spark.ml.feature.RandomProjection
+import org.apache.spark.ml.feature.BucketedRandomProjectionLSH
 import org.apache.spark.ml.linalg.Vectors
 // $example off$
 import org.apache.spark.sql.SparkSession
 
-object RandomProjectionExample {
+object BucketedRandomProjectionLSHExample {
   def main(args: Array[String]): Unit = {
     // Creates a SparkSession
     val spark = SparkSession
       .builder
-      .appName("RandomProjectionExample")
+      .appName("BucketedRandomProjectionLSHExample")
       .getOrCreate()
 
     // $example on$
@@ -40,9 +40,9 @@ object RandomProjectionExample {
       (2, Vectors.dense(-1.0, 1.0))
     )).toDF("id", "keys")
 
-    val rp = new RandomProjection()
+    val rp = new BucketedRandomProjectionLSH()
       .setBucketLength(2.0)
-      .setOutputDim(1)
+      .setNumHashTables(1)
       .setInputCol("keys")
       .setOutputCol("values")
 

examples/src/main/scala/org/apache/spark/examples/ml/LSHTransformationExample.scala

@@ -19,7 +19,7 @@
 package org.apache.spark.examples.ml
 
 // $example on$
-import org.apache.spark.ml.feature.MinHash
+import org.apache.spark.ml.feature.MinHashLSH
 import org.apache.spark.ml.linalg.Vectors
 // $example off$
 import org.apache.spark.sql.SparkSession
@@ -40,17 +40,16 @@ object LSHTransformationExample {
     )).toDF("id", "keys")
 
     // Single LSH hashing
-    val mhSingleHash = new MinHash()
-      .setOutputDim(1)
+    val mhSingleHash = new MinHashLSH()
       .setInputCol("keys")
       .setOutputCol("values")
     val modelSingleHash = mhSingleHash.fit(dataFrame)
     // Feature transformation: add a new column for a hash value
     modelSingleHash.transform(dataFrame).show()
 
-    // Use more than 1 hash functions
-    val mh = new MinHash()
-      .setOutputDim(5)
+    // Use more than 1 hash tables
+    val mh = new MinHashLSH()
+      .setNumHashTables(5)
       .setInputCol("keys")
       .setOutputCol("values")
     val model = mh.fit(dataFrame)