From 2609c9c69228666a5aa0bd544beaf4a039951a01 Mon Sep 17 00:00:00 2001 From: hongdongdong Date: Mon, 4 Mar 2019 16:43:56 +0800 Subject: [PATCH 01/11] [SPARK-26929][SQL]fix table owner use user instead of principal when create table through spark-sql or beeline --- .../apache/spark/sql/hive/client/HiveClientImpl.scala | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 2ca54afa31e6..ceffdd10daa5 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -17,11 +17,12 @@ package org.apache.spark.sql.hive.client -import java.io.{File, PrintStream} +import java.io.{File, IOException, PrintStream} import java.lang.{Iterable => JIterable} import java.util.{Locale, Map => JMap} import java.util.concurrent.TimeUnit._ +import javax.security.auth.login.LoginException import scala.collection.JavaConverters._ import scala.collection.mutable import scala.collection.mutable.ArrayBuffer @@ -55,6 +56,7 @@ import org.apache.spark.sql.hive.HiveExternalCatalog.{DATASOURCE_SCHEMA, DATASOU import org.apache.spark.sql.hive.client.HiveClientImpl._ import org.apache.spark.sql.types._ import org.apache.spark.util.{CircularBuffer, Utils} +import org.apache.hadoop.hive.shims.{Utils => HiveUtils} /** * A class that wraps the HiveClient and converts its responses to externally visible classes. @@ -191,7 +193,12 @@ private[hive] class HiveClientImpl( /** Returns the configuration for the current session. */ def conf: HiveConf = state.getConf - private val userName = conf.getUser + private val userName: String = try { + val ugi = HiveUtils.getUGI + ugi.getShortUserName + } catch { + case e: LoginException => throw new IOException(e) + } override def getConf(key: String, defaultValue: String): String = { conf.get(key, defaultValue) From 05e54fec41832282f3712ecaa0e1b2c1400c60b1 Mon Sep 17 00:00:00 2001 From: hongdd Date: Thu, 25 Jul 2019 14:06:26 +0800 Subject: [PATCH 02/11] Update HiveClientImpl.scala Modify exception --- .../scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index ceffdd10daa5..609bc6ccf26e 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -197,7 +197,7 @@ private[hive] class HiveClientImpl( val ugi = HiveUtils.getUGI ugi.getShortUserName } catch { - case e: LoginException => throw new IOException(e) + case _: LoginException => throw new IOException("Can not get login user.") } override def getConf(key: String, defaultValue: String): String = { From bdcc9fa4ec20a19dfc099fd90b01014dffea6616 Mon Sep 17 00:00:00 2001 From: hongdd Date: Thu, 25 Jul 2019 14:49:04 +0800 Subject: [PATCH 03/11] Update HiveClientImpl.scala --- .../org/apache/spark/sql/hive/client/HiveClientImpl.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 609bc6ccf26e..7a3dbf7e4483 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -23,6 +23,7 @@ import java.util.{Locale, Map => JMap} import java.util.concurrent.TimeUnit._ import javax.security.auth.login.LoginException + import scala.collection.JavaConverters._ import scala.collection.mutable import scala.collection.mutable.ArrayBuffer @@ -39,6 +40,7 @@ import org.apache.hadoop.hive.ql.metadata.{Hive, Partition => HivePartition, Tab import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.HIVE_COLUMN_ORDER_ASC import org.apache.hadoop.hive.ql.processors._ import org.apache.hadoop.hive.ql.session.SessionState +import org.apache.hadoop.hive.shims.{Utils => HiveUtils} import org.apache.spark.{SparkConf, SparkException} import org.apache.spark.internal.Logging @@ -56,7 +58,6 @@ import org.apache.spark.sql.hive.HiveExternalCatalog.{DATASOURCE_SCHEMA, DATASOU import org.apache.spark.sql.hive.client.HiveClientImpl._ import org.apache.spark.sql.types._ import org.apache.spark.util.{CircularBuffer, Utils} -import org.apache.hadoop.hive.shims.{Utils => HiveUtils} /** * A class that wraps the HiveClient and converts its responses to externally visible classes. From e8c05712db5dc95ab3482e3c663255a8b734e67e Mon Sep 17 00:00:00 2001 From: hongdd Date: Thu, 25 Jul 2019 15:09:24 +0800 Subject: [PATCH 04/11] Update HiveClientImpl.scala --- .../scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 859a96dbff5c..9fd535ecd255 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -21,7 +21,6 @@ import java.io.{File, IOException, PrintStream} import java.lang.{Iterable => JIterable} import java.util.{Locale, Map => JMap} import java.util.concurrent.TimeUnit._ - import javax.security.auth.login.LoginException import scala.collection.JavaConverters._ From c27d4affbd3371b55b197c63cc8f72e0b59613bc Mon Sep 17 00:00:00 2001 From: hongdd Date: Thu, 25 Jul 2019 16:03:57 +0800 Subject: [PATCH 05/11] Update HiveClientImpl.scala conflict of hiveUtils --- .../org/apache/spark/sql/hive/client/HiveClientImpl.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 9fd535ecd255..4ae1ac5a4608 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -42,7 +42,7 @@ import org.apache.hadoop.hive.ql.session.SessionState import org.apache.hadoop.hive.serde.serdeConstants import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe import org.apache.hadoop.hive.serde2.`lazy`.LazySimpleSerDe -import org.apache.hadoop.hive.shims.{Utils => HiveUtils} +import org.apache.hadoop.hive.shims.{Utils => HiveShimsUtils} import org.apache.spark.{SparkConf, SparkException} import org.apache.spark.internal.Logging @@ -223,7 +223,7 @@ private[hive] class HiveClientImpl( } private val userName: String = try { - val ugi = HiveUtils.getUGI + val ugi = HiveShimsUtils.getUGI ugi.getShortUserName } catch { case _: LoginException => throw new IOException("Can not get login user.") From f1eb8cfe9c32e74822b7701588d6e961d0d767ea Mon Sep 17 00:00:00 2001 From: hongdd Date: Fri, 26 Jul 2019 10:10:35 +0800 Subject: [PATCH 06/11] Update HiveClientImpl.scala --- .../spark/sql/hive/client/HiveClientImpl.scala | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 4ae1ac5a4608..50bf17d07eaa 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.hive.client -import java.io.{File, IOException, PrintStream} +import java.io.{File, PrintStream} import java.lang.{Iterable => JIterable} import java.util.{Locale, Map => JMap} import java.util.concurrent.TimeUnit._ @@ -42,7 +42,7 @@ import org.apache.hadoop.hive.ql.session.SessionState import org.apache.hadoop.hive.serde.serdeConstants import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe import org.apache.hadoop.hive.serde2.`lazy`.LazySimpleSerDe -import org.apache.hadoop.hive.shims.{Utils => HiveShimsUtils} +import org.apache.hadoop.security.UserGroupInformation import org.apache.spark.{SparkConf, SparkException} import org.apache.spark.internal.Logging @@ -222,11 +222,16 @@ private[hive] class HiveClientImpl( hiveConf } - private val userName: String = try { - val ugi = HiveShimsUtils.getUGI + private val userName = try { + val doAs = sys.env.get("HADOOP_USER_NAME").orNull + val ugi = if (doAs != null && doAs.length() > 0) { + UserGroupInformation.createProxyUser(doAs, UserGroupInformation.getLoginUser()) + } else { + UserGroupInformation.getCurrentUser + } ugi.getShortUserName } catch { - case _: LoginException => throw new IOException("Can not get login user.") + case _: LoginException => throw new LoginException("Can not get login user.") } override def getConf(key: String, defaultValue: String): String = { From 0c7ddb1b4fabde6ca757514f896e7ae9170c9079 Mon Sep 17 00:00:00 2001 From: hongdd Date: Tue, 10 Sep 2019 13:48:45 +0800 Subject: [PATCH 07/11] Update HiveClientImpl.scala --- .../org/apache/spark/sql/hive/client/HiveClientImpl.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 50bf17d07eaa..bd975b61d8e4 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -231,7 +231,9 @@ private[hive] class HiveClientImpl( } ugi.getShortUserName } catch { - case _: LoginException => throw new LoginException("Can not get login user.") + case e: Exception => + logWarning("Can not get login user.") + throw e } override def getConf(key: String, defaultValue: String): String = { From 60115989886001dc0a1ded86b6f0fd5f6bb1bc92 Mon Sep 17 00:00:00 2001 From: hongdd Date: Tue, 10 Sep 2019 13:49:11 +0800 Subject: [PATCH 08/11] Update HiveClientImpl.scala --- .../scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index bd975b61d8e4..05b97ba095b3 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -21,7 +21,6 @@ import java.io.{File, PrintStream} import java.lang.{Iterable => JIterable} import java.util.{Locale, Map => JMap} import java.util.concurrent.TimeUnit._ -import javax.security.auth.login.LoginException import scala.collection.JavaConverters._ import scala.collection.mutable From 76192b22a1778b203489c629e007bb299e552e0c Mon Sep 17 00:00:00 2001 From: hongdd Date: Tue, 10 Sep 2019 17:27:34 +0800 Subject: [PATCH 09/11] Update HiveClientImpl.scala --- .../scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 05b97ba095b3..d8069dc81916 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -231,7 +231,7 @@ private[hive] class HiveClientImpl( ugi.getShortUserName } catch { case e: Exception => - logWarning("Can not get login user.") + logError("Can not get login user.") throw e } From a8d6b1d69b7cc6b5cc788714616a31d886db799f Mon Sep 17 00:00:00 2001 From: hongdd Date: Wed, 11 Sep 2019 09:26:24 +0800 Subject: [PATCH 10/11] Update HiveClientImpl.scala --- .../org/apache/spark/sql/hive/client/HiveClientImpl.scala | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index d8069dc81916..1817d779c2a0 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -221,7 +221,7 @@ private[hive] class HiveClientImpl( hiveConf } - private val userName = try { + private val userName = { val doAs = sys.env.get("HADOOP_USER_NAME").orNull val ugi = if (doAs != null && doAs.length() > 0) { UserGroupInformation.createProxyUser(doAs, UserGroupInformation.getLoginUser()) @@ -229,10 +229,6 @@ private[hive] class HiveClientImpl( UserGroupInformation.getCurrentUser } ugi.getShortUserName - } catch { - case e: Exception => - logError("Can not get login user.") - throw e } override def getConf(key: String, defaultValue: String): String = { From 2f37513969306b89eb9f7650bc3d082b44705114 Mon Sep 17 00:00:00 2001 From: hongdd Date: Thu, 12 Sep 2019 09:40:57 +0800 Subject: [PATCH 11/11] Update HiveClientImpl.scala --- .../apache/spark/sql/hive/client/HiveClientImpl.scala | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 1817d779c2a0..ab92597e7a10 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -221,15 +221,7 @@ private[hive] class HiveClientImpl( hiveConf } - private val userName = { - val doAs = sys.env.get("HADOOP_USER_NAME").orNull - val ugi = if (doAs != null && doAs.length() > 0) { - UserGroupInformation.createProxyUser(doAs, UserGroupInformation.getLoginUser()) - } else { - UserGroupInformation.getCurrentUser - } - ugi.getShortUserName - } + private val userName = UserGroupInformation.getCurrentUser.getShortUserName override def getConf(key: String, defaultValue: String): String = { conf.get(key, defaultValue)