From ac373cf882819941b65a596d36c0e601f2e2735e Mon Sep 17 00:00:00 2001 From: Khushboo Bhatia Date: Tue, 28 May 2019 17:43:04 -0400 Subject: [PATCH 01/12] Use a custom executor service if config is set --- .../GlueMetastoreClientDelegate.java | 22 ++----- .../catalog/util/ThreadExecutorFactory.java | 61 +++++++++++++++++++ .../GlueMetastoreClientDelegateTest.java | 5 +- 3 files changed, 70 insertions(+), 18 deletions(-) create mode 100644 aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/util/ThreadExecutorFactory.java diff --git a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java index 31ffcab9..0477e4d1 100644 --- a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java +++ b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java @@ -4,10 +4,7 @@ import com.amazonaws.glue.catalog.converters.CatalogToHiveConverter; import com.amazonaws.glue.catalog.converters.GlueInputConverter; import com.amazonaws.glue.catalog.converters.HiveToCatalogConverter; -import com.amazonaws.glue.catalog.util.BatchCreatePartitionsHelper; -import com.amazonaws.glue.catalog.util.ExpressionHelper; -import com.amazonaws.glue.catalog.util.MetastoreClientUtils; -import com.amazonaws.glue.catalog.util.PartitionKey; +import com.amazonaws.glue.catalog.util.*; import com.amazonaws.glue.shims.AwsGlueHiveShims; import com.amazonaws.glue.shims.ShimsLoader; import com.amazonaws.services.glue.AWSGlue; @@ -157,15 +154,6 @@ public class GlueMetastoreClientDelegate { */ public static final int GET_PARTITIONS_MAX_SIZE = 1000; - private static final int NUM_EXECUTOR_THREADS = 5; - static final String GLUE_METASTORE_DELEGATE_THREADPOOL_NAME_FORMAT = "glue-metastore-delegate-%d"; - private static final ExecutorService GLUE_METASTORE_DELEGATE_THREAD_POOL = Executors.newFixedThreadPool( - NUM_EXECUTOR_THREADS, - new ThreadFactoryBuilder() - .setNameFormat(GLUE_METASTORE_DELEGATE_THREADPOOL_NAME_FORMAT) - .setDaemon(true).build() - ); - /** * Maximum number of Glue Segments. A segment defines a non-overlapping region of a table's partitions, * allowing multiple requests to be executed in parallel. @@ -179,6 +167,7 @@ public class GlueMetastoreClientDelegate { */ public static final int MAX_NUM_PARTITION_SEGMENTS = 10; + private final ExecutorService executorService; private final AWSGlue glueClient; private final HiveConf conf; private final Warehouse wh; @@ -200,6 +189,7 @@ public GlueMetastoreClientDelegate(HiveConf conf, AWSGlue glueClient, Warehouse this.conf = conf; this.glueClient = glueClient; this.wh = wh; + this.executorService = ThreadExecutorFactory.getGlueMetastoreDelegateThreadPool(conf); // TODO - May be validate catalogId confirms to AWS AccountId too. catalogId = MetastoreClientUtils.getCatalogId(conf); } @@ -740,7 +730,7 @@ private List batchCreatePartitions( int j = Math.min(i + BATCH_CREATE_PARTITIONS_MAX_REQUEST_SIZE, catalogPartitions.size()); final List partitionsOnePage = catalogPartitions.subList(i, j); - batchCreatePartitionsFutures.add(GLUE_METASTORE_DELEGATE_THREAD_POOL.submit(new Callable() { + batchCreatePartitionsFutures.add(this.executorService.submit(new Callable() { @Override public BatchCreatePartitionsHelper call() throws Exception { return new BatchCreatePartitionsHelper(glueClient, dbName, tableName, catalogId, partitionsOnePage, ifNotExists) @@ -868,7 +858,7 @@ public List getPartitionsByNames .withTableName(tableName) .withPartitionsToGet(batch) .withCatalogId(catalogId); - batchGetPartitionFutures.add(GLUE_METASTORE_DELEGATE_THREAD_POOL.submit(new Callable() { + batchGetPartitionFutures.add(this.executorService.submit(new Callable() { @Override public BatchGetPartitionResult call() throws Exception { return glueClient.batchGetPartition(request); @@ -974,7 +964,7 @@ private List getPartitionsParallel( // We could convert this into a parallelStream after upgrading to JDK 8 compiler base. List>> futures = Lists.newArrayList(); for (final Segment segment : segments) { - futures.add(GLUE_METASTORE_DELEGATE_THREAD_POOL.submit(new Callable>() { + futures.add(this.executorService.submit(new Callable>() { @Override public List call() throws Exception { return getCatalogPartitions(databaseName, tableName, expression, max, segment); diff --git a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/util/ThreadExecutorFactory.java b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/util/ThreadExecutorFactory.java new file mode 100644 index 00000000..eeece723 --- /dev/null +++ b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/util/ThreadExecutorFactory.java @@ -0,0 +1,61 @@ +package com.amazonaws.glue.catalog.util; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.hive.conf.HiveConf; +import + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; + +public class ThreadExecutorFactory { + public static final int NUM_EXECUTOR_THREADS = 5; + public static final String GLUE_METASTORE_DELEGATE_THREADPOOL_NAME_FORMAT = "glue-metastore-delegate-%d"; + public static final String CUSTOM_EXECUTOR_CONF = "hive.metastore.custom.executor.class"; + + private static ExecutorService delegateThreadPool = null; + + public static ExecutorService getGlueMetastoreDelegateThreadPool(HiveConf conf) { + if (delegateThreadPool == null ) { + synchronized (delegateThreadPool) { + + if (delegateThreadPool == null ) { + ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setNameFormat(GLUE_METASTORE_DELEGATE_THREADPOOL_NAME_FORMAT) + .setDaemon(true).build(); + String customExecutorClass = conf.get(CUSTOM_EXECUTOR_CONF); + if (customExecutorClass != null && !customExecutorClass.isEmpty()){ + delegateThreadPool = (ExecutorService) getInstanceByReflection( + customExecutorClass, NUM_EXECUTOR_THREADS, threadFactory); + } else { + delegateThreadPool = Executors.newFixedThreadPool(NUM_EXECUTOR_THREADS, threadFactory); + } + } + } + } + + return delegateThreadPool; + } + + public static Object getInstanceByReflection(String className, int arg1, ThreadFactory arg2) { + try { + Class[] argTypes = {Integer.class, java.util.concurrent.ThreadFactory.class}; + Class classDefinition = Class.forName(className); + Constructor cons = classDefinition.getConstructor(argTypes); + Object[] args = {arg1, arg2}; + return cons.newInstance(args); + } catch( ClassNotFoundException ce ){ + + } catch ( NoSuchMethodException ne ) { + + } catch( InstantiationException ie ) { + + } catch (IllegalAccessException iae ){ + + } catch( InvocationTargetException ite ){ + + } + } +} diff --git a/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegateTest.java b/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegateTest.java index 11bdc454..38db2a0a 100644 --- a/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegateTest.java +++ b/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegateTest.java @@ -3,6 +3,7 @@ import com.amazonaws.glue.catalog.converters.CatalogToHiveConverter; import com.amazonaws.glue.catalog.converters.GlueInputConverter; import com.amazonaws.glue.catalog.util.TestObjects; +import com.amazonaws.glue.catalog.util.ThreadExecutorFactory; import com.amazonaws.services.glue.AWSGlue; import com.amazonaws.services.glue.model.AlreadyExistsException; import com.amazonaws.services.glue.model.BatchCreatePartitionRequest; @@ -1521,9 +1522,9 @@ public void testUpdateTableColumnStatistics() throws Exception { private void assertDaemonThreadPools() { String threadNameCreatePrefix = - GlueMetastoreClientDelegate.GLUE_METASTORE_DELEGATE_THREADPOOL_NAME_FORMAT.substring( + ThreadExecutorFactory.GLUE_METASTORE_DELEGATE_THREADPOOL_NAME_FORMAT.substring( 0, - GlueMetastoreClientDelegate.GLUE_METASTORE_DELEGATE_THREADPOOL_NAME_FORMAT.indexOf('%')); + ThreadExecutorFactory.GLUE_METASTORE_DELEGATE_THREADPOOL_NAME_FORMAT.indexOf('%')); for (Thread thread : Thread.getAllStackTraces().keySet()) { String threadName = thread.getName(); if (threadName != null && threadName.startsWith(threadNameCreatePrefix)) { From 90dcc79363de54fd3a1a2b2aacf081bf125c00e0 Mon Sep 17 00:00:00 2001 From: Khushboo Bhatia Date: Tue, 28 May 2019 17:54:34 -0400 Subject: [PATCH 02/12] Fixed null check for threadPool --- .../com/amazonaws/glue/catalog/util/ThreadExecutorFactory.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/util/ThreadExecutorFactory.java b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/util/ThreadExecutorFactory.java index eeece723..b0fac003 100644 --- a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/util/ThreadExecutorFactory.java +++ b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/util/ThreadExecutorFactory.java @@ -29,7 +29,8 @@ public static ExecutorService getGlueMetastoreDelegateThreadPool(HiveConf conf) if (customExecutorClass != null && !customExecutorClass.isEmpty()){ delegateThreadPool = (ExecutorService) getInstanceByReflection( customExecutorClass, NUM_EXECUTOR_THREADS, threadFactory); - } else { + } + if ( delegateThreadPool == null ) { delegateThreadPool = Executors.newFixedThreadPool(NUM_EXECUTOR_THREADS, threadFactory); } } From ce2529980ea94cf670ab03091eb6a5f5d6a3703b Mon Sep 17 00:00:00 2001 From: Khushboo Bhatia Date: Wed, 29 May 2019 15:33:02 -0400 Subject: [PATCH 03/12] Removed singleton logic for custom executor and logs --- .../GlueMetastoreClientDelegate.java | 25 ++++++++- .../catalog/util/ThreadExecutorFactory.java | 56 +++++++------------ .../GlueMetastoreClientDelegateTest.java | 5 +- 3 files changed, 44 insertions(+), 42 deletions(-) diff --git a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java index 0477e4d1..d4b1a75a 100644 --- a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java +++ b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java @@ -4,7 +4,11 @@ import com.amazonaws.glue.catalog.converters.CatalogToHiveConverter; import com.amazonaws.glue.catalog.converters.GlueInputConverter; import com.amazonaws.glue.catalog.converters.HiveToCatalogConverter; -import com.amazonaws.glue.catalog.util.*; +import com.amazonaws.glue.catalog.util.BatchCreatePartitionsHelper; +import com.amazonaws.glue.catalog.util.ExpressionHelper; +import com.amazonaws.glue.catalog.util.MetastoreClientUtils; +import com.amazonaws.glue.catalog.util.PartitionKey; +import com.amazonaws.glue.catalog.util.ThreadExecutorFactory; import com.amazonaws.glue.shims.AwsGlueHiveShims; import com.amazonaws.glue.shims.ShimsLoader; import com.amazonaws.services.glue.AWSGlue; @@ -52,8 +56,8 @@ import com.google.common.base.Throwables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.StatsSetupConst; @@ -117,6 +121,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.ThreadFactory; import java.util.regex.Pattern; import static com.amazonaws.glue.catalog.converters.ConverterUtils.stringToCatalogTable; @@ -154,6 +159,14 @@ public class GlueMetastoreClientDelegate { */ public static final int GET_PARTITIONS_MAX_SIZE = 1000; + private static final int NUM_EXECUTOR_THREADS = 5; + static final String GLUE_METASTORE_DELEGATE_THREADPOOL_NAME_FORMAT = "glue-metastore-delegate-%d"; + private static final ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setNameFormat(GLUE_METASTORE_DELEGATE_THREADPOOL_NAME_FORMAT) + .setDaemon(true).build(); + private static final ExecutorService GLUE_METASTORE_DELEGATE_THREAD_POOL = Executors.newFixedThreadPool( + NUM_EXECUTOR_THREADS, threadFactory); + /** * Maximum number of Glue Segments. A segment defines a non-overlapping region of a table's partitions, * allowing multiple requests to be executed in parallel. @@ -189,7 +202,13 @@ public GlueMetastoreClientDelegate(HiveConf conf, AWSGlue glueClient, Warehouse this.conf = conf; this.glueClient = glueClient; this.wh = wh; - this.executorService = ThreadExecutorFactory.getGlueMetastoreDelegateThreadPool(conf); + ExecutorService customExecutor = ThreadExecutorFactory.getCustomThreadPool(conf, NUM_EXECUTOR_THREADS, threadFactory); + if (customExecutor != null ) { + this.executorService = customExecutor; + } else { + this.executorService = GLUE_METASTORE_DELEGATE_THREAD_POOL; + } + // TODO - May be validate catalogId confirms to AWS AccountId too. catalogId = MetastoreClientUtils.getCatalogId(conf); } diff --git a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/util/ThreadExecutorFactory.java b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/util/ThreadExecutorFactory.java index b0fac003..a644d244 100644 --- a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/util/ThreadExecutorFactory.java +++ b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/util/ThreadExecutorFactory.java @@ -1,42 +1,25 @@ package com.amazonaws.glue.catalog.util; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.amazonaws.glue.catalog.metastore.AWSGlueClientFactory; import org.apache.hadoop.hive.conf.HiveConf; -import +import org.apache.log4j.Logger; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; + import java.util.concurrent.ThreadFactory; public class ThreadExecutorFactory { - public static final int NUM_EXECUTOR_THREADS = 5; - public static final String GLUE_METASTORE_DELEGATE_THREADPOOL_NAME_FORMAT = "glue-metastore-delegate-%d"; + private static final Logger logger = Logger.getLogger(ThreadExecutorFactory.class); public static final String CUSTOM_EXECUTOR_CONF = "hive.metastore.custom.executor.class"; - private static ExecutorService delegateThreadPool = null; - - public static ExecutorService getGlueMetastoreDelegateThreadPool(HiveConf conf) { - if (delegateThreadPool == null ) { - synchronized (delegateThreadPool) { - - if (delegateThreadPool == null ) { - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setNameFormat(GLUE_METASTORE_DELEGATE_THREADPOOL_NAME_FORMAT) - .setDaemon(true).build(); - String customExecutorClass = conf.get(CUSTOM_EXECUTOR_CONF); - if (customExecutorClass != null && !customExecutorClass.isEmpty()){ - delegateThreadPool = (ExecutorService) getInstanceByReflection( - customExecutorClass, NUM_EXECUTOR_THREADS, threadFactory); - } - if ( delegateThreadPool == null ) { - delegateThreadPool = Executors.newFixedThreadPool(NUM_EXECUTOR_THREADS, threadFactory); - } - } + public static ExecutorService getCustomThreadPool(HiveConf conf, int numThreads, ThreadFactory threadFactory) { + ExecutorService delegateThreadPool = null; + String customExecutorClass = conf.get(CUSTOM_EXECUTOR_CONF); + if (customExecutorClass != null && !customExecutorClass.isEmpty()){ + delegateThreadPool = (ExecutorService) getInstanceByReflection(customExecutorClass, numThreads, threadFactory); } - } - return delegateThreadPool; } @@ -47,16 +30,17 @@ public static Object getInstanceByReflection(String className, int arg1, ThreadF Constructor cons = classDefinition.getConstructor(argTypes); Object[] args = {arg1, arg2}; return cons.newInstance(args); - } catch( ClassNotFoundException ce ){ - - } catch ( NoSuchMethodException ne ) { - - } catch( InstantiationException ie ) { - - } catch (IllegalAccessException iae ){ - - } catch( InvocationTargetException ite ){ - + } catch( ClassNotFoundException e ){ + logger.warn("Exception in initializing custom executor ",e); + } catch ( NoSuchMethodException e ) { + logger.warn("Exception in initializing custom executor ", e); + } catch( InstantiationException e ) { + logger.warn("Exception in initializing custom executor ", e); + } catch (IllegalAccessException e ){ + logger.warn("Exception in initializing custom executor ", e); + } catch( InvocationTargetException e ){ + logger.warn("Exception in initializing custom executor ", e); } + return null; } } diff --git a/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegateTest.java b/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegateTest.java index 38db2a0a..11bdc454 100644 --- a/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegateTest.java +++ b/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegateTest.java @@ -3,7 +3,6 @@ import com.amazonaws.glue.catalog.converters.CatalogToHiveConverter; import com.amazonaws.glue.catalog.converters.GlueInputConverter; import com.amazonaws.glue.catalog.util.TestObjects; -import com.amazonaws.glue.catalog.util.ThreadExecutorFactory; import com.amazonaws.services.glue.AWSGlue; import com.amazonaws.services.glue.model.AlreadyExistsException; import com.amazonaws.services.glue.model.BatchCreatePartitionRequest; @@ -1522,9 +1521,9 @@ public void testUpdateTableColumnStatistics() throws Exception { private void assertDaemonThreadPools() { String threadNameCreatePrefix = - ThreadExecutorFactory.GLUE_METASTORE_DELEGATE_THREADPOOL_NAME_FORMAT.substring( + GlueMetastoreClientDelegate.GLUE_METASTORE_DELEGATE_THREADPOOL_NAME_FORMAT.substring( 0, - ThreadExecutorFactory.GLUE_METASTORE_DELEGATE_THREADPOOL_NAME_FORMAT.indexOf('%')); + GlueMetastoreClientDelegate.GLUE_METASTORE_DELEGATE_THREADPOOL_NAME_FORMAT.indexOf('%')); for (Thread thread : Thread.getAllStackTraces().keySet()) { String threadName = thread.getName(); if (threadName != null && threadName.startsWith(threadNameCreatePrefix)) { From 90a6d23c10159c11b3dbd19a21939c02f892a000 Mon Sep 17 00:00:00 2001 From: Khushboo Bhatia Date: Thu, 30 May 2019 00:25:02 -0400 Subject: [PATCH 04/12] Use an interface for thread executor factory --- .../metastore/ExecutorServiceFactory.java | 12 +++++ .../GlueMetastoreClientDelegate.java | 49 ++++++++++++++----- .../metastore/ThreadPoolExecutorFactory.java | 22 +++++++++ .../catalog/util/ThreadExecutorFactory.java | 46 ----------------- 4 files changed, 70 insertions(+), 59 deletions(-) create mode 100644 aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/ExecutorServiceFactory.java create mode 100644 aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/ThreadPoolExecutorFactory.java delete mode 100644 aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/util/ThreadExecutorFactory.java diff --git a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/ExecutorServiceFactory.java b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/ExecutorServiceFactory.java new file mode 100644 index 00000000..6168a01f --- /dev/null +++ b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/ExecutorServiceFactory.java @@ -0,0 +1,12 @@ +package com.amazonaws.glue.catalog.metastore; + +import org.apache.hadoop.hive.conf.HiveConf; + +import java.util.concurrent.ExecutorService; + +/* + * Interface for creating an ExecutorService + */ +public interface ExecutorServiceFactory { + public ExecutorService getExecutorService(HiveConf conf); +} diff --git a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java index d4b1a75a..fc74335c 100644 --- a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java +++ b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java @@ -110,6 +110,8 @@ import org.apache.thrift.TException; import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; @@ -159,14 +161,10 @@ public class GlueMetastoreClientDelegate { */ public static final int GET_PARTITIONS_MAX_SIZE = 1000; - private static final int NUM_EXECUTOR_THREADS = 5; - static final String GLUE_METASTORE_DELEGATE_THREADPOOL_NAME_FORMAT = "glue-metastore-delegate-%d"; - private static final ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setNameFormat(GLUE_METASTORE_DELEGATE_THREADPOOL_NAME_FORMAT) - .setDaemon(true).build(); - private static final ExecutorService GLUE_METASTORE_DELEGATE_THREAD_POOL = Executors.newFixedThreadPool( - NUM_EXECUTOR_THREADS, threadFactory); + public static final String CUSTOM_EXECUTOR_FACTORY_CONF = "hive.metastore.executorservice.factory.class"; + static final String GLUE_METASTORE_DELEGATE_THREADPOOL_NAME_FORMAT = "glue-metastore-delegate-%d"; + /** * Maximum number of Glue Segments. A segment defines a non-overlapping region of a table's partitions, * allowing multiple requests to be executed in parallel. @@ -191,6 +189,36 @@ public class GlueMetastoreClientDelegate { public static final String CATALOG_ID_CONF = "hive.metastore.glue.catalogid"; public static final String NUM_PARTITION_SEGMENTS_CONF = "aws.glue.partition.num.segments"; + private ExecutorService getExecutorService(HiveConf conf) throws MetaException { + + String customFactoryClassName = conf.get(CUSTOM_EXECUTOR_FACTORY_CONF); + ExecutorServiceFactory factory = null; + if (!StringUtils.isEmpty(customFactoryClassName)) { + try { + factory = (ExecutorServiceFactory) Class.forName(customFactoryClassName).newInstance(); + } catch (ClassNotFoundException e) { + String msg = "Could not load class defined in conf " + CUSTOM_EXECUTOR_FACTORY_CONF; + logger.warn(msg); + throw new MetaException(msg + e); + } catch (InstantiationException e) { + String msg = "Could not instantiate class defined in conf " + CUSTOM_EXECUTOR_FACTORY_CONF; + logger.warn(msg); + throw new MetaException(msg + e); + } catch (IllegalAccessException e) { + String msg = "Could not instantiate class defined in conf " + CUSTOM_EXECUTOR_FACTORY_CONF; + logger.warn(msg); + throw new MetaException(msg + e); + } + } else { + // If custom factory conf is not defined or if there is an error in loading it via reflection + factory = new ThreadPoolExecutorFactory(); + } + + return factory.getExecutorService(conf); + } + + + public GlueMetastoreClientDelegate(HiveConf conf, AWSGlue glueClient, Warehouse wh) throws MetaException { checkNotNull(conf, "Hive Config cannot be null"); checkNotNull(glueClient, "glueClient cannot be null"); @@ -202,12 +230,7 @@ public GlueMetastoreClientDelegate(HiveConf conf, AWSGlue glueClient, Warehouse this.conf = conf; this.glueClient = glueClient; this.wh = wh; - ExecutorService customExecutor = ThreadExecutorFactory.getCustomThreadPool(conf, NUM_EXECUTOR_THREADS, threadFactory); - if (customExecutor != null ) { - this.executorService = customExecutor; - } else { - this.executorService = GLUE_METASTORE_DELEGATE_THREAD_POOL; - } + this.executorService = getExecutorService(conf); // TODO - May be validate catalogId confirms to AWS AccountId too. catalogId = MetastoreClientUtils.getCatalogId(conf); diff --git a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/ThreadPoolExecutorFactory.java b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/ThreadPoolExecutorFactory.java new file mode 100644 index 00000000..92273d58 --- /dev/null +++ b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/ThreadPoolExecutorFactory.java @@ -0,0 +1,22 @@ +package com.amazonaws.glue.catalog.metastore; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.hive.conf.HiveConf; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +i +public class ThreadPoolExecutorFactory implements ExecutorServiceFactory { + private static final int NUM_EXECUTOR_THREADS = 5; + + private static final ExecutorService GLUE_METASTORE_DELEGATE_THREAD_POOL = Executors.newFixedThreadPool( + NUM_EXECUTOR_THREADS, new ThreadFactoryBuilder() + .setNameFormat(GlueMetastoreClientDelegate.GLUE_METASTORE_DELEGATE_THREADPOOL_NAME_FORMAT) + .setDaemon(true).build() + ); + + @Override + public ExecutorService getExecutorService(HiveConf conf) { + return GLUE_METASTORE_DELEGATE_THREAD_POOL; + } +} diff --git a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/util/ThreadExecutorFactory.java b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/util/ThreadExecutorFactory.java deleted file mode 100644 index a644d244..00000000 --- a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/util/ThreadExecutorFactory.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.amazonaws.glue.catalog.util; - -import com.amazonaws.glue.catalog.metastore.AWSGlueClientFactory; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.log4j.Logger; - -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.util.concurrent.ExecutorService; - -import java.util.concurrent.ThreadFactory; - -public class ThreadExecutorFactory { - private static final Logger logger = Logger.getLogger(ThreadExecutorFactory.class); - public static final String CUSTOM_EXECUTOR_CONF = "hive.metastore.custom.executor.class"; - - public static ExecutorService getCustomThreadPool(HiveConf conf, int numThreads, ThreadFactory threadFactory) { - ExecutorService delegateThreadPool = null; - String customExecutorClass = conf.get(CUSTOM_EXECUTOR_CONF); - if (customExecutorClass != null && !customExecutorClass.isEmpty()){ - delegateThreadPool = (ExecutorService) getInstanceByReflection(customExecutorClass, numThreads, threadFactory); - } - return delegateThreadPool; - } - - public static Object getInstanceByReflection(String className, int arg1, ThreadFactory arg2) { - try { - Class[] argTypes = {Integer.class, java.util.concurrent.ThreadFactory.class}; - Class classDefinition = Class.forName(className); - Constructor cons = classDefinition.getConstructor(argTypes); - Object[] args = {arg1, arg2}; - return cons.newInstance(args); - } catch( ClassNotFoundException e ){ - logger.warn("Exception in initializing custom executor ",e); - } catch ( NoSuchMethodException e ) { - logger.warn("Exception in initializing custom executor ", e); - } catch( InstantiationException e ) { - logger.warn("Exception in initializing custom executor ", e); - } catch (IllegalAccessException e ){ - logger.warn("Exception in initializing custom executor ", e); - } catch( InvocationTargetException e ){ - logger.warn("Exception in initializing custom executor ", e); - } - return null; - } -} From 88635bf1b5546c6952d753f069d642d2277b46d2 Mon Sep 17 00:00:00 2001 From: Khushboo Bhatia Date: Thu, 30 May 2019 00:31:59 -0400 Subject: [PATCH 05/12] Use an interface for thread executor factory --- .../catalog/metastore/GlueMetastoreClientDelegate.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java index fc74335c..fdbda484 100644 --- a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java +++ b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java @@ -8,7 +8,6 @@ import com.amazonaws.glue.catalog.util.ExpressionHelper; import com.amazonaws.glue.catalog.util.MetastoreClientUtils; import com.amazonaws.glue.catalog.util.PartitionKey; -import com.amazonaws.glue.catalog.util.ThreadExecutorFactory; import com.amazonaws.glue.shims.AwsGlueHiveShims; import com.amazonaws.glue.shims.ShimsLoader; import com.amazonaws.services.glue.AWSGlue; @@ -57,7 +56,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.StatsSetupConst; @@ -110,8 +108,6 @@ import org.apache.thrift.TException; import java.io.IOException; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; @@ -121,9 +117,7 @@ import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.Future; -import java.util.concurrent.ThreadFactory; import java.util.regex.Pattern; import static com.amazonaws.glue.catalog.converters.ConverterUtils.stringToCatalogTable; @@ -164,7 +158,7 @@ public class GlueMetastoreClientDelegate { public static final String CUSTOM_EXECUTOR_FACTORY_CONF = "hive.metastore.executorservice.factory.class"; static final String GLUE_METASTORE_DELEGATE_THREADPOOL_NAME_FORMAT = "glue-metastore-delegate-%d"; - + /** * Maximum number of Glue Segments. A segment defines a non-overlapping region of a table's partitions, * allowing multiple requests to be executed in parallel. From 9f24b1b7b7d3f3c119196c2c403684b6f6113a92 Mon Sep 17 00:00:00 2001 From: Khushboo Bhatia Date: Thu, 30 May 2019 00:34:22 -0400 Subject: [PATCH 06/12] Use an interface for thread executor factory --- .../glue/catalog/metastore/ThreadPoolExecutorFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/ThreadPoolExecutorFactory.java b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/ThreadPoolExecutorFactory.java index 92273d58..86c255ae 100644 --- a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/ThreadPoolExecutorFactory.java +++ b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/ThreadPoolExecutorFactory.java @@ -5,7 +5,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -i + public class ThreadPoolExecutorFactory implements ExecutorServiceFactory { private static final int NUM_EXECUTOR_THREADS = 5; From 7398ba896ce3816c66da4f862c4600d2c4a7e598 Mon Sep 17 00:00:00 2001 From: Khushboo Bhatia Date: Thu, 30 May 2019 00:57:42 -0400 Subject: [PATCH 07/12] Call factory by reflection --- .../GlueMetastoreClientDelegate.java | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java index fdbda484..f85c9183 100644 --- a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java +++ b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java @@ -108,6 +108,8 @@ import org.apache.thrift.TException; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; @@ -186,29 +188,23 @@ public class GlueMetastoreClientDelegate { private ExecutorService getExecutorService(HiveConf conf) throws MetaException { String customFactoryClassName = conf.get(CUSTOM_EXECUTOR_FACTORY_CONF); - ExecutorServiceFactory factory = null; + ExecutorService executorService = null; if (!StringUtils.isEmpty(customFactoryClassName)) { try { - factory = (ExecutorServiceFactory) Class.forName(customFactoryClassName).newInstance(); - } catch (ClassNotFoundException e) { - String msg = "Could not load class defined in conf " + CUSTOM_EXECUTOR_FACTORY_CONF; - logger.warn(msg); - throw new MetaException(msg + e); - } catch (InstantiationException e) { - String msg = "Could not instantiate class defined in conf " + CUSTOM_EXECUTOR_FACTORY_CONF; - logger.warn(msg); - throw new MetaException(msg + e); - } catch (IllegalAccessException e) { - String msg = "Could not instantiate class defined in conf " + CUSTOM_EXECUTOR_FACTORY_CONF; - logger.warn(msg); - throw new MetaException(msg + e); + Class clazz = Class.forName(customFactoryClassName); + Object factory = clazz.newInstance(); + Method method = clazz.getDeclaredMethod("getExecutorService", HiveConf.class); + executorService = (ExecutorService)method.invoke(factory, conf); + } catch(Exception e) { + logger.warn(e.getMessage()); + throw new MetaException(e.getMessage() + e); } } else { // If custom factory conf is not defined or if there is an error in loading it via reflection - factory = new ThreadPoolExecutorFactory(); + executorService = new ThreadPoolExecutorFactory().getExecutorService(conf); } - return factory.getExecutorService(conf); + return executorService; } From df44fa3a6d0539054e0e3417643b4a9c1e6b4a42 Mon Sep 17 00:00:00 2001 From: Khushboo Bhatia Date: Thu, 30 May 2019 15:49:58 -0400 Subject: [PATCH 08/12] Use ReflectionUtils to load instance --- ...ava => DefaultExecutorServiceFactory.java} | 2 +- .../GlueMetastoreClientDelegate.java | 33 +++++-------------- 2 files changed, 10 insertions(+), 25 deletions(-) rename aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/{ThreadPoolExecutorFactory.java => DefaultExecutorServiceFactory.java} (90%) diff --git a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/ThreadPoolExecutorFactory.java b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/DefaultExecutorServiceFactory.java similarity index 90% rename from aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/ThreadPoolExecutorFactory.java rename to aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/DefaultExecutorServiceFactory.java index 86c255ae..c1643323 100644 --- a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/ThreadPoolExecutorFactory.java +++ b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/DefaultExecutorServiceFactory.java @@ -6,7 +6,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -public class ThreadPoolExecutorFactory implements ExecutorServiceFactory { +public class DefaultExecutorServiceFactory implements ExecutorServiceFactory { private static final int NUM_EXECUTOR_THREADS = 5; private static final ExecutorService GLUE_METASTORE_DELEGATE_THREAD_POOL = Executors.newFixedThreadPool( diff --git a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java index f85c9183..fc4f5d6b 100644 --- a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java +++ b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java @@ -104,12 +104,11 @@ import org.apache.hadoop.hive.metastore.api.UnknownTableException; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; +import org.apache.hadoop.util.ReflectionUtils; import org.apache.log4j.Logger; import org.apache.thrift.TException; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; @@ -185,30 +184,16 @@ public class GlueMetastoreClientDelegate { public static final String CATALOG_ID_CONF = "hive.metastore.glue.catalogid"; public static final String NUM_PARTITION_SEGMENTS_CONF = "aws.glue.partition.num.segments"; - private ExecutorService getExecutorService(HiveConf conf) throws MetaException { - - String customFactoryClassName = conf.get(CUSTOM_EXECUTOR_FACTORY_CONF); - ExecutorService executorService = null; - if (!StringUtils.isEmpty(customFactoryClassName)) { - try { - Class clazz = Class.forName(customFactoryClassName); - Object factory = clazz.newInstance(); - Method method = clazz.getDeclaredMethod("getExecutorService", HiveConf.class); - executorService = (ExecutorService)method.invoke(factory, conf); - } catch(Exception e) { - logger.warn(e.getMessage()); - throw new MetaException(e.getMessage() + e); - } - } else { - // If custom factory conf is not defined or if there is an error in loading it via reflection - executorService = new ThreadPoolExecutorFactory().getExecutorService(conf); - } - - return executorService; + private ExecutorService getExecutorService(HiveConf conf) { + Class executorFactoryClass = conf + .getClass(CUSTOM_EXECUTOR_FACTORY_CONF, + DefaultExecutorServiceFactory.class).asSubclass( + ExecutorServiceFactory.class); + ExecutorServiceFactory factory = ReflectionUtils.newInstance( + executorFactoryClass, conf); + return factory.getExecutorService(conf); } - - public GlueMetastoreClientDelegate(HiveConf conf, AWSGlue glueClient, Warehouse wh) throws MetaException { checkNotNull(conf, "Hive Config cannot be null"); checkNotNull(glueClient, "glueClient cannot be null"); From 54154a834ef453f5bae777a897293ed0f9c58ccf Mon Sep 17 00:00:00 2001 From: Khushboo Bhatia Date: Mon, 3 Jun 2019 16:57:05 -0400 Subject: [PATCH 09/12] Added unit tests --- .../GlueMetastoreClientDelegate.java | 6 ++-- .../GlueMetastoreClientDelegateTest.java | 30 +++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java index fc4f5d6b..b3728644 100644 --- a/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java +++ b/aws-glue-datacatalog-client-common/src/main/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegate.java @@ -184,8 +184,8 @@ public class GlueMetastoreClientDelegate { public static final String CATALOG_ID_CONF = "hive.metastore.glue.catalogid"; public static final String NUM_PARTITION_SEGMENTS_CONF = "aws.glue.partition.num.segments"; - private ExecutorService getExecutorService(HiveConf conf) { - Class executorFactoryClass = conf + protected ExecutorService getExecutorService() { + Class executorFactoryClass = this.conf .getClass(CUSTOM_EXECUTOR_FACTORY_CONF, DefaultExecutorServiceFactory.class).asSubclass( ExecutorServiceFactory.class); @@ -205,7 +205,7 @@ public GlueMetastoreClientDelegate(HiveConf conf, AWSGlue glueClient, Warehouse this.conf = conf; this.glueClient = glueClient; this.wh = wh; - this.executorService = getExecutorService(conf); + this.executorService = getExecutorService(); // TODO - May be validate catalogId confirms to AWS AccountId too. catalogId = MetastoreClientUtils.getCatalogId(conf); diff --git a/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegateTest.java b/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegateTest.java index 11bdc454..c6b77c26 100644 --- a/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegateTest.java +++ b/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegateTest.java @@ -54,6 +54,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; @@ -79,6 +80,7 @@ import java.util.Date; import java.util.List; import java.util.Set; +import java.util.concurrent.*; import static com.amazonaws.glue.catalog.util.TestObjects.getTestDatabase; import static com.amazonaws.glue.catalog.util.TestObjects.getTestPartition; @@ -153,6 +155,34 @@ private void setupMockWarehouseForPath(Path path, boolean isDir, boolean mkDir) when(wh.mkdirs(path, true)).thenReturn(mkDir); } + // ===================== Thread Executor ===================== + + private class TestExecutorService extends ScheduledThreadPoolExecutor { + + public TestExecutorService(int corePoolSize, ThreadFactory factory) { + super(corePoolSize, factory); + } + } + + private class TestExecutorFactory implements ExecutorServiceFactory { + private final ExecutorService execService = new TestExecutorService(1, new ThreadFactoryBuilder().build()); + + @Override + public ExecutorService getExecutorService(HiveConf conf) { + return execService; + } + } + + @Test + public void testExecutorService() throws Exception { + Object defaultExecutorService = new DefaultExecutorServiceFactory().getExecutorService(conf); + assertEquals("Default executor service should be used", metastoreClientDelegate.getExecutorService(), defaultExecutorService); + conf.set(GlueMetastoreClientDelegate.CUSTOM_EXECUTOR_FACTORY_CONF, TestExecutorFactory.class.getName()); + GlueMetastoreClientDelegate customDelegate = new GlueMetastoreClientDelegate(conf, glueClient, wh); + Object customExecutorService = new TestExecutorFactory().getExecutorService(conf); + assertEquals("Custom executor service should be used", customDelegate.getExecutorService(), customExecutorService); + } + // ===================== Database ===================== @Test From e9952021986010b3c356c26288a87189b3820285 Mon Sep 17 00:00:00 2001 From: Khushboo Bhatia Date: Tue, 4 Jun 2019 10:48:38 -0400 Subject: [PATCH 10/12] Updated unit tests --- .../GlueMetastoreClientDelegateTest.java | 27 ++++++------------- .../catalog/util/TestExecutorService.java | 11 ++++++++ .../util/TestExecutorServiceFactory.java | 20 ++++++++++++++ 3 files changed, 39 insertions(+), 19 deletions(-) create mode 100644 aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/util/TestExecutorService.java create mode 100644 aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/util/TestExecutorServiceFactory.java diff --git a/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegateTest.java b/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegateTest.java index c6b77c26..9b51754c 100644 --- a/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegateTest.java +++ b/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegateTest.java @@ -3,6 +3,8 @@ import com.amazonaws.glue.catalog.converters.CatalogToHiveConverter; import com.amazonaws.glue.catalog.converters.GlueInputConverter; import com.amazonaws.glue.catalog.util.TestObjects; +import com.amazonaws.glue.catalog.util.TestExecutorService; +import com.amazonaws.glue.catalog.util.TestExecutorServiceFactory; import com.amazonaws.services.glue.AWSGlue; import com.amazonaws.services.glue.model.AlreadyExistsException; import com.amazonaws.services.glue.model.BatchCreatePartitionRequest; @@ -157,29 +159,16 @@ private void setupMockWarehouseForPath(Path path, boolean isDir, boolean mkDir) // ===================== Thread Executor ===================== - private class TestExecutorService extends ScheduledThreadPoolExecutor { - - public TestExecutorService(int corePoolSize, ThreadFactory factory) { - super(corePoolSize, factory); - } - } - - private class TestExecutorFactory implements ExecutorServiceFactory { - private final ExecutorService execService = new TestExecutorService(1, new ThreadFactoryBuilder().build()); - - @Override - public ExecutorService getExecutorService(HiveConf conf) { - return execService; - } - } - @Test public void testExecutorService() throws Exception { Object defaultExecutorService = new DefaultExecutorServiceFactory().getExecutorService(conf); assertEquals("Default executor service should be used", metastoreClientDelegate.getExecutorService(), defaultExecutorService); - conf.set(GlueMetastoreClientDelegate.CUSTOM_EXECUTOR_FACTORY_CONF, TestExecutorFactory.class.getName()); - GlueMetastoreClientDelegate customDelegate = new GlueMetastoreClientDelegate(conf, glueClient, wh); - Object customExecutorService = new TestExecutorFactory().getExecutorService(conf); + HiveConf customConf = new HiveConf(); + customConf.set(GlueMetastoreClientDelegate.CATALOG_ID_CONF, CATALOG_ID); + customConf.setClass(GlueMetastoreClientDelegate.CUSTOM_EXECUTOR_FACTORY_CONF, TestExecutorServiceFactory.class, ExecutorServiceFactory.class); + GlueMetastoreClientDelegate customDelegate = new GlueMetastoreClientDelegate(customConf, mock(AWSGlue.class), mock(Warehouse.class)); + Object customExecutorService = new TestExecutorServiceFactory().getExecutorService(customConf); + assertEquals("Custom executor service should be used", customDelegate.getExecutorService(), customExecutorService); } diff --git a/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/util/TestExecutorService.java b/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/util/TestExecutorService.java new file mode 100644 index 00000000..d6f8f28b --- /dev/null +++ b/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/util/TestExecutorService.java @@ -0,0 +1,11 @@ +package com.amazonaws.glue.catalog.util; + +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadFactory; + +public class TestExecutorService extends ScheduledThreadPoolExecutor { + + public TestExecutorService(int corePoolSize, ThreadFactory factory) { + super(corePoolSize, factory); + } +} \ No newline at end of file diff --git a/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/util/TestExecutorServiceFactory.java b/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/util/TestExecutorServiceFactory.java new file mode 100644 index 00000000..541d8614 --- /dev/null +++ b/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/util/TestExecutorServiceFactory.java @@ -0,0 +1,20 @@ +package com.amazonaws.glue.catalog.util; + +import com.amazonaws.glue.catalog.metastore.ExecutorServiceFactory; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.hive.conf.HiveConf; + +import java.util.concurrent.ExecutorService; + +public class TestExecutorServiceFactory implements ExecutorServiceFactory { + private static ExecutorService execService = new TestExecutorService(1, new ThreadFactoryBuilder().build()); + + @Override + public ExecutorService getExecutorService(HiveConf conf) { + return execService; + } + + +} + + From 1bb77a82e1d61835546b164e5246883273bd8750 Mon Sep 17 00:00:00 2001 From: Khushboo Bhatia Date: Tue, 4 Jun 2019 10:59:50 -0400 Subject: [PATCH 11/12] cleaning up --- .../catalog/metastore/GlueMetastoreClientDelegateTest.java | 3 --- .../glue/catalog/util/TestExecutorServiceFactory.java | 2 -- 2 files changed, 5 deletions(-) diff --git a/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegateTest.java b/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegateTest.java index 9b51754c..8e9c1e82 100644 --- a/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegateTest.java +++ b/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/metastore/GlueMetastoreClientDelegateTest.java @@ -3,7 +3,6 @@ import com.amazonaws.glue.catalog.converters.CatalogToHiveConverter; import com.amazonaws.glue.catalog.converters.GlueInputConverter; import com.amazonaws.glue.catalog.util.TestObjects; -import com.amazonaws.glue.catalog.util.TestExecutorService; import com.amazonaws.glue.catalog.util.TestExecutorServiceFactory; import com.amazonaws.services.glue.AWSGlue; import com.amazonaws.services.glue.model.AlreadyExistsException; @@ -56,7 +55,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; @@ -82,7 +80,6 @@ import java.util.Date; import java.util.List; import java.util.Set; -import java.util.concurrent.*; import static com.amazonaws.glue.catalog.util.TestObjects.getTestDatabase; import static com.amazonaws.glue.catalog.util.TestObjects.getTestPartition; diff --git a/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/util/TestExecutorServiceFactory.java b/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/util/TestExecutorServiceFactory.java index 541d8614..668bf0f0 100644 --- a/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/util/TestExecutorServiceFactory.java +++ b/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/util/TestExecutorServiceFactory.java @@ -13,8 +13,6 @@ public class TestExecutorServiceFactory implements ExecutorServiceFactory { public ExecutorService getExecutorService(HiveConf conf) { return execService; } - - } From e8401cced7fad37a904a97dd1ee55903d01f7b3e Mon Sep 17 00:00:00 2001 From: Khushboo Bhatia Date: Tue, 4 Jun 2019 11:04:10 -0400 Subject: [PATCH 12/12] Removing extra empty lines --- .../amazonaws/glue/catalog/util/TestExecutorServiceFactory.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/util/TestExecutorServiceFactory.java b/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/util/TestExecutorServiceFactory.java index 668bf0f0..c55aba8d 100644 --- a/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/util/TestExecutorServiceFactory.java +++ b/aws-glue-datacatalog-client-common/src/test/java/com/amazonaws/glue/catalog/util/TestExecutorServiceFactory.java @@ -14,5 +14,3 @@ public ExecutorService getExecutorService(HiveConf conf) { return execService; } } - -