Change 2.2.0 to 2.3.0

Author: HyukjinKwon

python/pyspark/sql/readwriter.py

@@ -142,7 +142,7 @@ def options(self, **options):
             self.option(k, options[k])
         return self
 
-    @since(2.2)
+    @since(2.3)
     def unsetOption(self, key):
         """Un-sets the option given to the key for the underlying data source.
         """
@@ -583,7 +583,7 @@ def options(self, **options):
             self.option(k, options[k])
         return self
 
-    @since(2.2)
+    @since(2.3)
     def unsetOption(self, key):
         """Un-sets the option given to the key for the underlying data source.
         """

python/pyspark/sql/streaming.py

@@ -374,11 +374,11 @@ def options(self, **options):
             self.option(k, options[k])
         return self
 
-    @since(2.2)
+    @since(2.3)
     def unsetOption(self, key):
         """Un-sets the option given to the key for the underlying data source.
 
-        .. note:: Experimental.
+        .. note:: Evolving.
         """
         self._jreader = self._jreader.unsetOption(key)
         return self
@@ -748,11 +748,11 @@ def options(self, **options):
             self.option(k, options[k])
         return self
 
-    @since(2.2)
+    @since(2.3)
     def unsetOption(self, key):
         """Un-sets the option given to the key for the underlying data source.
 
-        .. note:: Experimental.
+        .. note:: Evolving.
         """
         self._jwrite = self._jwrite.unsetOption(key)
         return self

sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala

@@ -122,7 +122,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
   /**
    * (Scala-specific) Adds an input option for the underlying data source.
    *
-   * @since 2.2.0
+   * @since 2.3.0
    */
   def option(key: String, value: Seq[String]): DataFrameReader = {
     option(key, compact(render(value)))
@@ -131,7 +131,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
   /**
    * Adds an input option for the underlying data source.
    *
-   * @since 2.2.0
+   * @since 2.3.0
    */
   def option(key: String, value: Array[String]): DataFrameReader = option(key, value.toSeq)
 
@@ -170,7 +170,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
   /**
    * Un-sets an input option for the underlying data source.
    *
-   * @since 2.2.0
+   * @since 2.3.0
    */
   def unsetOption(key: String): DataFrameReader = {
     this.extraOptions.remove(key)

sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala

@@ -131,7 +131,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   /**
    * (Scala-specific) Adds an output option for the underlying data source.
    *
-   * @since 2.2.0
+   * @since 2.3.0
    */
   def option(key: String, value: Seq[String]): DataFrameWriter[T] = {
     option(key, compact(render(value)))
@@ -140,7 +140,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   /**
    * Adds an output option for the underlying data source.
    *
-   * @since 2.2.0
+   * @since 2.3.0
    */
   def option(key: String, value: Array[String]): DataFrameWriter[T] = option(key, value.toSeq)
 
@@ -179,7 +179,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   /**
    * Un-sets an output option for the underlying data source.
    *
-   * @since 2.2.0
+   * @since 2.3.0
    */
   def unsetOption(key: String): DataFrameWriter[T] = {
     this.extraOptions.remove(key)

sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala

@@ -114,7 +114,7 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
   /**
    * (Scala-specific) Adds an input option for the underlying data source.
    *
-   * @since 2.2.0
+   * @since 2.3.0
    */
   def option(key: String, value: Seq[String]): DataStreamReader = {
     option(key, compact(render(value)))
@@ -123,7 +123,7 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
   /**
    * Adds an input option for the underlying data source.
    *
-   * @since 2.2.0
+   * @since 2.3.0
    */
   def option(key: String, value: Array[String]): DataStreamReader = option(key, value.toSeq)
 
@@ -162,7 +162,7 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
   /**
    * Un-sets an input option for the underlying data source.
    *
-   * @since 2.2.0
+   * @since 2.3.0
    */
   def unsetOption(key: String): DataStreamReader = {
     this.extraOptions.remove(key)

sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala

@@ -186,7 +186,7 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
   /**
    * (Scala-specific) Adds an output option for the underlying data source.
    *
-   * @since 2.2.0
+   * @since 2.3.0
    */
   def option(key: String, value: Seq[String]): DataStreamWriter[T] = {
     option(key, compact(render(value)))
@@ -195,7 +195,7 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
   /**
    * Adds an output option for the underlying data source.
    *
-   * @since 2.2.0
+   * @since 2.3.0
    */
   def option(key: String, value: Array[String]): DataStreamWriter[T] = option(key, value.toSeq)
 
@@ -234,7 +234,7 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
   /**
    * Un-sets an option option for the underlying data source.
    *
-   * @since 2.2.0
+   * @since 2.3.0
    */
   def unsetOption(key: String): DataStreamWriter[T] = {
     this.extraOptions.remove(key)

sql/hive/src/test/scala/org/apache/spark/sql/sources/SimpleTextHadoopFsRelationSuite.scala

@@ -65,8 +65,9 @@ class SimpleTextHadoopFsRelationSuite extends HadoopFsRelationTest with Predicat
 
   test("test hadoop conf option propagation") {
     withTempPath { file =>
-      // Test write side
       val nullVal: String = null
+
+      // Test write side
       val df = spark.range(10).selectExpr("cast(id as string)")
       df.write
         .option("some-random-write-option", "hahah-WRITE")