minor changes addressing review comments

Author: tomasatdatabricks

python/sparkdl/image/imageIO.py

@@ -40,7 +40,7 @@
 #  NOTE: likely to be migrated to Spark ImageSchema code in the near future.
 _OcvType = namedtuple("OcvType",["name","ord","nChannels","dtype"])
 
-#  NOTE: likely to be migrated to Spark ImageSchema code in the near future.
+
 _supportedOcvTypes = (
     _OcvType(name="CV_8UC1",  ord=0,  nChannels=1, dtype="uint8"  ),
     _OcvType(name="CV_32FC1", ord=5,  nChannels=1, dtype="float32"),
@@ -50,18 +50,19 @@
     _OcvType(name="CV_32FC4", ord=29, nChannels=4, dtype="float32"),
 )
 
-__ocvTypesByName = {m.name:m for m in _supportedOcvTypes}
-__ocvTypesByOrdinal = {m.ord:m for m in _supportedOcvTypes}
+#  NOTE: likely to be migrated to Spark ImageSchema code in the near future.
+_ocvTypesByName = {m.name:m for m in _supportedOcvTypes}
+_ocvTypesByOrdinal = {m.ord:m for m in _supportedOcvTypes}
 
 def imageTypeByOrdinal(ord):
-    if not ord in __ocvTypesByOrdinal:
+    if not ord in _ocvTypesByOrdinal:
         raise KeyError("unsupported image type with ordinal %d, supported OpenCV types = %s"  % (ord,str(_supportedOcvTypes)))
-    return __ocvTypesByOrdinal[ord]
+    return _ocvTypesByOrdinal[ord]
 
 def imageTypeByName(name):
-    if not name in __ocvTypesByName:
+    if not name in _ocvTypesByName:
         raise KeyError("unsupported image type with name '%s', supported supported OpenCV types = %s" % (name,str(_supportedOcvTypes)))
-    return __ocvTypesByName[name]
+    return _ocvTypesByName[name]
 
 def imageArrayToStruct(imgArray,origin=""):
     """

python/tests/image/test_imageIO.py

@@ -111,21 +111,6 @@ def _test(array):
             _test(np.random.randint(0, 256, (10, 11, nChannels), 'uint8'))
             _test(np.random.random_sample((10,11,nChannels)).astype('float32'))
 
-    def test_image_round_trip(self):
-        # Test round trip: array -> png -> sparkImg -> array
-        binarySchema = StructType([StructField("data", BinaryType(), False)])
-        df = self.session.createDataFrame([[bytearray(pngData)]], binarySchema)
-
-        # Convert to images
-        decImg = udf(lambda x:imageIO.imageArrayToStruct(imageIO.PIL_decode(x)), ImageSchema.imageSchema['image'].dataType)
-        imageDF = df.select(decImg("data").alias("image"))
-        row = imageDF.first()
-        # array comes out of PIL and is in RGB order
-        testArray = imageIO.imageStructToArray(row.image)
-        self.assertEqual(testArray.shape, array.shape)
-        self.assertEqual(testArray.dtype, array.dtype)
-        self.assertTrue(np.all(array == testArray))
-
     # read images now part of spark, no need to test it here
     def test_readImages(self):
         # Test that reading

python/tests/transformers/image_utils.py

@@ -92,19 +92,6 @@ def compareClassSets(self, preds1, preds2):
             self.assertEqual(set([v[1] for v in v1]), set([v[1] for v in preds2[k]]))
 
 
-def getSampleImageList():
-    imageFiles = glob(os.path.join(_getSampleJPEGDir(), "*"))
-    images = []
-    for f in imageFiles:
-        try:
-            img = PIL.Image.open(f)
-        except IOError:
-            warn("Could not read file in image directory.")
-            images.append(None)
-        else:
-            images.append(img)
-    return imageFiles, images
-
 
 def executeKerasInceptionV3(image_df, uri_col="filePath"):
     """

python/tests/transformers/named_image_test.py

@@ -30,8 +30,7 @@
 from sparkdl.image.image import ImageSchema
 
 from ..tests import SparkDLTestCase
-from .image_utils import getSampleImageDF, getSampleImageList
-
+from .image_utils import getSampleImageDF
 
 
 class KerasApplicationModelTestCase(SparkDLTestCase):
@@ -67,22 +66,29 @@ class NamedImageTransformerBaseTestCase(SparkDLTestCase):
     numPartitionsOverride = None
 
 
+    @classmethod
+    def getSampleImageList(cls):
+        imageFiles = glob(os.path.join(_getSampleJPEGDir(), "*"))
+        images = []
+        for f in imageFiles:
+            try:
+                img = PIL.Image.open(f)
+                shape = cls.appModel.inputShape()
+                images.append(imageIO._reverseChannels(np.array(img.resize(shape))))
+            except IOError:
+                warn("Could not read file in image directory.")
+                images.append(None)
+        return imageFiles, np.array(images)
+
+
     @classmethod
     def setUpClass(cls):
         super(NamedImageTransformerBaseTestCase, cls).setUpClass()
-
         cls.appModel = keras_apps.getKerasApplicationModel(cls.name)
-        shape = cls.appModel.inputShape()
-
-        imgFiles, images = getSampleImageList()
-        imageArray = np.empty((len(images), shape[0], shape[1], 3), 'uint8')
-        for i, img in enumerate(images):
-            assert img is not None and img.mode == "RGB"
-            imageArray[i] = imageIO._reverseChannels(np.array(img.resize(shape)))
+        imgFiles, imageArray = getSampleImageList()
         cls.imageArray = imageArray
         cls.imgFiles = imgFiles
         cls.fileOrder = {imgFiles[i].split('/')[-1]:i for i in range(len(imgFiles))}
-
         # Predict the class probabilities for the images in our test library using keras API
         # and cache for use by multiple tests.
         preppedImage = cls.appModel._testPreprocess(imageArray.astype('float32'))

python/tests/transformers/tf_image_test.py

@@ -37,14 +37,12 @@ class TFImageTransformerExamplesTest(SparkDLTestCase, ImageNetOutputComparisonTe
     # Test loading & pre-processing as an example of a simple graph
     # NOTE: resizing here/tensorflow and in keras workflow are different, so the
     # test would fail with resizing added in.
-
     def _loadImageViaKeras(self, raw_uri):
         uri = raw_uri[5:] if raw_uri.startswith("file:/") else raw_uri
         image = img_to_array(load_img(uri))
         image = np.expand_dims(image, axis=0)
         return preprocess_input(image)
 
-    # TODO: I believe this is already tested in named_image_test, should we remove it?
     def test_load_image_vs_keras(self):
         g = tf.Graph()
         with g.as_default():
@@ -70,7 +68,6 @@ def test_load_image_vs_keras(self):
             self.assertTrue( (processed == keras_processed).all() )
 
 
-            # TODO: I believe this is already tested in named_image_test, should we remove it?
     def test_load_image_vs_keras_RGB(self):
         g = tf.Graph()
         with g.as_default():