From 15b176d95e8a437e2e58fd73b2501575ed605508 Mon Sep 17 00:00:00 2001 From: Thomas Marquardt Date: Sun, 21 Jun 2020 05:14:15 +0000 Subject: [PATCH] SAXParser concurrency bug fix. DETAILS: There is a concurrency bug in the XML parsing logic which can cause the results of a List Blobs operation to appear empty. Please refer to Utility.java#L131. Utility.saxParserThreadLocal is a ThreadLocal with a SAXParserFactory member variable. The factory is initialized in ThreadLocal.initialValue which then continues to use the member variable reference. Because initialValue is called once for each thread, it is possible for Thread A to be after the call to "factory.setNamespaceAware(true)" but before the call to "return factory.newSAXParser()" when it yields to Thread B which calls "factory = SAXParserFactory.newInstance()" and then yields to Thread A which then calls "return factory.newSAXParser()". Since this is a reference to the factory member variable, this instance would therefore not be namespace aware since setNamespaceAware(true) has not been called. It therefore returns a SAXParser that is not namespace aware. A SAXParser that is not namespace aware will always return a list count of 0. Please refer to BlobListHandler.java#L84. BlobListHandler.startElement takes the localName parameter and compares it with "Blob" on line 84, to find each blob item in the list. When a SAXParser is not namespace aware, the localName parameter is always empty. Please refer to the documentation for DefaultHandler.startElement, which describes the localName parameter as follows, "The local name (without prefix), or the empty string if Namespace processing is not being performed." This was also confirmed by testing, so the documentation is correct. Thus when the SAXParser is not initialized correctly (is not namespace aware) the results of List Blobs operation will always appear to be empty. TESTS: The test testSAXParserConcurrency was added to validate that the SAXParser returned by Utility.getSAXParser is correctly configured, even when called under a highly concurrent load. This test will occassionaly fail without the fix, but the probability of faiulre is very low. To increease the likelihood of failure you can add calls to sleep before and after the call to "factory.setNamespaceAware(true)" in Utitlity.java as shown below: private static void sleep(long l) { try { Thread.sleep(l); } catch (InterruptedException e) { } } /** * Thread local for SAXParser. */ private static final ThreadLocal saxParserThreadLocal = new ThreadLocal() { SAXParserFactory factory; @Override public SAXParser initialValue() { factory = SAXParserFactory.newInstance(); sleep(100); factory.setNamespaceAware(true); sleep(10); try { return factory.newSAXParser(); } catch (SAXException e) { throw new RuntimeException("Unable to create SAXParser", e); } catch (ParserConfigurationException e) { throw new RuntimeException("Check parser configuration", e); } } }; --- .../storage/blob/CloudBlobContainerTests.java | 49 +++++++++++++++++++ .../microsoft/azure/storage/core/Utility.java | 3 +- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/CloudBlobContainerTests.java b/microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/CloudBlobContainerTests.java index 8e747d2c0..d46f1a0ea 100644 --- a/microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/CloudBlobContainerTests.java +++ b/microsoft-azure-storage-test/src/com/microsoft/azure/storage/blob/CloudBlobContainerTests.java @@ -22,8 +22,14 @@ import java.net.URISyntaxException; import java.security.InvalidKeyException; import java.util.*; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import javax.xml.parsers.SAXParser; import com.microsoft.azure.storage.*; +import com.microsoft.azure.storage.core.Utility; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -541,6 +547,49 @@ public void testCloudBlobContainerListBlobs() throws StorageException, IOExcepti assertTrue(blobNames.size() == 0); } + @Test + @Category({DevFabricTests.class, DevStoreTests.class}) + public void testSAXParserConcurrency() throws Exception { + final int totalCount = 200000; + final int numThreads = 200; + final AtomicInteger currentCount = new AtomicInteger(0); + final AtomicInteger pending = new AtomicInteger(0); + final AtomicInteger failureCount = new AtomicInteger(0); + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(numThreads); + + do { + final int count = currentCount.incrementAndGet(); + pending.incrementAndGet(); + executor.execute(new Runnable() { + @Override + public void run() { + pending.decrementAndGet(); + if (count > totalCount) { + return; + } + try { + SAXParser parser = Utility.getSAXParser(); + if (!parser.isNamespaceAware()) { + failureCount.incrementAndGet(); + } + assertEquals(true, parser.isNamespaceAware()); + } catch (Exception e) { + fail(e.toString()); + } + } + }); + + assertEquals(0, failureCount.get()); + + while (pending.get() > numThreads * 2) { + Thread.sleep(10); + } + } while (currentCount.get() < totalCount); + executor.shutdown(); + executor.awaitTermination(1, TimeUnit.MINUTES); + executor.shutdownNow(); + } + /** * List the blobs in a container with a prefix * diff --git a/microsoft-azure-storage/src/com/microsoft/azure/storage/core/Utility.java b/microsoft-azure-storage/src/com/microsoft/azure/storage/core/Utility.java index c82db7f63..3c809f5ff 100644 --- a/microsoft-azure-storage/src/com/microsoft/azure/storage/core/Utility.java +++ b/microsoft-azure-storage/src/com/microsoft/azure/storage/core/Utility.java @@ -129,9 +129,8 @@ protected DateFormat initialValue() { * Thread local for SAXParser. */ private static final ThreadLocal saxParserThreadLocal = new ThreadLocal() { - SAXParserFactory factory; @Override public SAXParser initialValue() { - factory = SAXParserFactory.newInstance(); + SAXParserFactory factory = SAXParserFactory.newInstance(); factory.setNamespaceAware(true); try { return factory.newSAXParser();