Feature extraction

src/python/turicreate/test/test_image_classifier.py

@@ -19,6 +19,7 @@
     _raise_error_if_not_sframe,
     _raise_error_if_not_sarray,
 )
+from turicreate.toolkits.image_analysis.image_analysis import MODEL_TO_FEATURE_SIZE_MAPPING, get_deep_features
 
 from . import util as test_util
 
@@ -73,7 +74,10 @@ def get_test_data():
         images.append(tc_image)
 
     labels = ["white"] * 5 + ["black"] * 5
-    return tc.SFrame({"awesome_image": images, "awesome_label": labels})
+    data_dict = {"awesome_image": images, "awesome_label": labels}
+    data = tc.SFrame(data_dict)
+
+    return data
 
 
 data = get_test_data()
@@ -84,19 +88,24 @@ class ImageClassifierTest(unittest.TestCase):
     def setUpClass(
         self,
         model="resnet-50",
+        feature="awesome_image", 
         input_image_shape=(3, 224, 224),
         tol=0.02,
         num_examples=100,
         label_type=int,
     ):
-        self.feature = "awesome_image"
+        self.feature = feature
         self.target = "awesome_label"
         self.input_image_shape = input_image_shape
         self.pre_trained_model = model
         self.tolerance = tol
 
+        # Get deep features if needed
+        if self.feature.endswith("WithDeepFeature"):
+            data[self.feature] = get_deep_features(data["awesome_image"], self.feature.split('_WithDeepFeature')[0])
+
         self.model = tc.image_classifier.create(
-            data, target=self.target, model=self.pre_trained_model, seed=42
+            data, target=self.target, feature=self.feature, model=self.pre_trained_model, seed=42
         )
         self.nn_model = self.model.feature_extractor
         self.lm_model = self.model.classifier
@@ -132,14 +141,13 @@ def assertListAlmostEquals(self, list1, list2, tol):
             self.assertAlmostEqual(a, b, delta=tol)
 
     def test_create_with_missing_value(self):
+        data_dict = {}
+        for col_name, col_type in zip(data.column_names(), data.column_types()):
+            data_dict[col_name] = tc.SArray([None], dtype=col_type)
         data_with_none = data.append(
-            tc.SFrame(
-                {
-                    self.feature: tc.SArray([None], dtype=tc.Image),
-                    self.target: [data[self.target][0]],
-                }
-            )
+            tc.SFrame(data_dict)
         )
+
         with self.assertRaises(_ToolkitError):
             tc.image_classifier.create(
                 data_with_none, feature=self.feature, target=self.target
@@ -161,6 +169,16 @@ def test_create_with_empty_dataset(self):
         with self.assertRaises(_ToolkitError):
             tc.image_classifier.create(data[:0], target=self.target)
 
+    def test_select_correct_feature_column_to_train(self):
+        # sending both, the correct extracted features colum and image column
+        if self.feature == "awesome_image":
+            test_data = data.select_columns([self.feature, self.target])
+            deep_features_col_name = self.pre_trained_model+"_WithDeepFeature"
+            test_data[deep_features_col_name] = get_deep_features(data["awesome_image"], 
+                self.pre_trained_model)
+            test_model = tc.image_classifier.create(test_data, target=self.target, model=self.pre_trained_model)
+            self.assertTrue(test_model.feature == deep_features_col_name)
+
     def test_predict(self):
         model = self.model
         for output_type in ["class", "probability_vector"]:
@@ -235,22 +253,26 @@ def test_export_coreml_predict(self):
         self.model.export_coreml(filename)
 
         coreml_model = coremltools.models.MLModel(filename)
-        img = data[0:1][self.feature][0]
-        img_fixed = tc.image_analysis.resize(img, *reversed(self.input_image_shape))
-        from PIL import Image
-
-        pil_img = Image.fromarray(img_fixed.pixel_data)
-
-        if _mac_ver() >= (10, 13):
-            classes = self.model.classifier.classes
-            ret = coreml_model.predict({self.feature: pil_img})
-            coreml_values = [ret[self.target + "Probability"][l] for l in classes]
-
-            self.assertListAlmostEquals(
-                coreml_values,
-                list(self.model.predict(img_fixed, output_type="probability_vector")),
-                self.tolerance,
-            )
+        if self.feature == "awesome_image":
+            img = data[0:1][self.feature][0]
+            img_fixed = tc.image_analysis.resize(img, *reversed(self.input_image_shape))
+            from PIL import Image
+
+            pil_img = Image.fromarray(img_fixed.pixel_data)
+
+            if _mac_ver() >= (10, 13):
+                classes = self.model.classifier.classes
+                ret = coreml_model.predict({self.feature: pil_img})
+                coreml_values = [ret[self.target + "Probability"][l] for l in classes]
+
+                self.assertListAlmostEquals(
+                    coreml_values,
+                    list(self.model.predict(img_fixed, output_type="probability_vector")),
+                    self.tolerance,
+                )
+        else:
+            # If the code came here that means the type of the feature used is deep_deatures and the predict fwature in coremltools doesn't work with deep_features yet so we will ignore this specific test case unitl the same is written.
+            pass
 
     def test_classify(self):
         model = self.model
@@ -333,6 +355,18 @@ def test_evaluate_explore(self):
         evaluation.explore()
 
 
+class ImageClassifierResnetTestWithDeepFeatures(ImageClassifierTest):
+    @classmethod
+    def setUpClass(self):
+        super(ImageClassifierResnetTestWithDeepFeatures, self).setUpClass(
+            model="resnet-50",
+            input_image_shape=(3, 224, 224),
+            tol=0.02,
+            num_examples=100,
+            feature="resnet-50_WithDeepFeature",
+        )
+
+
 class ImageClassifierSqueezeNetTest(ImageClassifierTest):
     @classmethod
     def setUpClass(self):
@@ -341,6 +375,18 @@ def setUpClass(self):
             input_image_shape=(3, 227, 227),
             tol=0.005,
             num_examples=200,
+            feature="awesome_image",
+        )
+
+class ImageClassifierSqueezeNetTestWithDeepFeatures(ImageClassifierTest):
+    @classmethod
+    def setUpClass(self):
+        super(ImageClassifierSqueezeNetTestWithDeepFeatures, self).setUpClass(
+            model="squeezenet_v1.1",
+            input_image_shape=(3, 227, 227),
+            tol=0.005,
+            num_examples=200,
+            feature="squeezenet_v1.1_WithDeepFeature",
         )
 
 
@@ -357,4 +403,22 @@ def setUpClass(self):
             tol=0.005,
             num_examples=100,
             label_type=str,
+            feature="awesome_image",
+        )
+
+
+# TODO: if on skip OS, test negative case
+@unittest.skipIf(
+    _mac_ver() < (10, 14), "VisionFeaturePrint_Scene only supported on macOS 10.14+"
+)
+class VisionFeaturePrintSceneTestWithDeepFeatures(ImageClassifierTest):
+    @classmethod
+    def setUpClass(self):
+        super(VisionFeaturePrintSceneTestWithDeepFeatures, self).setUpClass(
+            model="VisionFeaturePrint_Scene",
+            input_image_shape=(3, 299, 299),
+            tol=0.005,
+            num_examples=100,
+            label_type=str,
+            feature="VisionFeaturePrint_Scene_WithDeepFeature",
         )

src/python/turicreate/test/test_image_similarity.py

@@ -12,8 +12,11 @@
 from turicreate.toolkits._internal_utils import _mac_ver
 import tempfile
 from . import util as test_util
-import numpy as np
+
 from turicreate.toolkits._main import ToolkitError as _ToolkitError
+from turicreate.toolkits.image_analysis.image_analysis import MODEL_TO_FEATURE_SIZE_MAPPING, get_deep_features
+
+import numpy as np
 
 
 def get_test_data():
@@ -62,19 +65,22 @@ def get_test_data():
         )
         images.append(tc_image)
 
-    return tc.SFrame({"awesome_image": images})
+    data_dict = {"awesome_image": images}
+    data = tc.SFrame(data_dict)
+
+    return data
 
 
 data = get_test_data()
 
 
 class ImageSimilarityTest(unittest.TestCase):
     @classmethod
-    def setUpClass(self, input_image_shape=(3, 224, 224), model="resnet-50"):
+    def setUpClass(self, input_image_shape=(3, 224, 224), model="resnet-50", feature="awesome_image"):
         """
         The setup class method for the basic test case with all default values.
         """
-        self.feature = "awesome_image"
+        self.feature = feature
         self.label = None
         self.input_image_shape = input_image_shape
         self.pre_trained_model = model
@@ -85,6 +91,10 @@ def setUpClass(self, input_image_shape=(3, 224, 224), model="resnet-50"):
             "verbose": True,
         }
 
+        # Get deep features if needed
+        if self.feature.endswith("WithDeepFeature"):
+            data[self.feature] = get_deep_features(data["awesome_image"], self.feature.split('_WithDeepFeature')[0])
+
         # Model
         self.model = tc.image_similarity.create(
             data, feature=self.feature, label=None, model=self.pre_trained_model
@@ -251,21 +261,25 @@ def get_psnr(x, y):
         )
 
         # Get model distances for comparison
-        img = data[0:1][self.feature][0]
-        img_fixed = tc.image_analysis.resize(img, *reversed(self.input_image_shape))
-        tc_ret = self.model.query(img_fixed, k=data.num_rows())
-
-        if _mac_ver() >= (10, 13):
-            from PIL import Image as _PIL_Image
-
-            pil_img = _PIL_Image.fromarray(img_fixed.pixel_data)
-            coreml_ret = coreml_model.predict({"awesome_image": pil_img})
-
-            # Compare distances
-            coreml_distances = np.array(coreml_ret["distance"])
-            tc_distances = tc_ret.sort("reference_label")["distance"].to_numpy()
-            psnr_value = get_psnr(coreml_distances, tc_distances)
-            self.assertTrue(psnr_value > 50)
+        if self.feature == "awesome_image":
+            img = data[0:1][self.feature][0]
+            img_fixed = tc.image_analysis.resize(img, *reversed(self.input_image_shape))
+            tc_ret = self.model.query(img_fixed, k=data.num_rows())
+
+            if _mac_ver() >= (10, 13):
+                from PIL import Image as _PIL_Image
+
+                pil_img = _PIL_Image.fromarray(img_fixed.pixel_data)
+                coreml_ret = coreml_model.predict({"awesome_image": pil_img})
+
+                # Compare distances
+                coreml_distances = np.array(coreml_ret["distance"])
+                tc_distances = tc_ret.sort("reference_label")["distance"].to_numpy()
+                psnr_value = get_psnr(coreml_distances, tc_distances)
+                self.assertTrue(psnr_value > 50)
+        else:
+            # If the code came here that means the type of the feature used is deep_deatures and the predict fwature in coremltools doesn't work with deep_features yet so we will ignore this specific test case unitl the same is written.
+            pass
 
     def test_save_and_load(self):
         with test_util.TempDirectory() as filename:
@@ -287,11 +301,27 @@ def test_save_and_load(self):
             print("Export coreml passed")
 
 
+class ImageSimilarityResnetTestWithDeepFeatures(ImageSimilarityTest):
+    @classmethod
+    def setUpClass(self):
+        super(ImageSimilarityResnetTestWithDeepFeatures, self).setUpClass(
+            model="resnet-50", input_image_shape=(3, 224, 224), feature="resnet-50_WithDeepFeature"
+        )
+
+
 class ImageSimilaritySqueezeNetTest(ImageSimilarityTest):
     @classmethod
     def setUpClass(self):
         super(ImageSimilaritySqueezeNetTest, self).setUpClass(
-            model="squeezenet_v1.1", input_image_shape=(3, 227, 227)
+            model="squeezenet_v1.1", input_image_shape=(3, 227, 227), feature="awesome_image"
+        )
+
+
+class ImageSimilaritySqueezeNetTestWithDeepFeatures(ImageSimilarityTest):
+    @classmethod
+    def setUpClass(self):
+        super(ImageSimilaritySqueezeNetTestWithDeepFeatures, self).setUpClass(
+            model="squeezenet_v1.1", input_image_shape=(3, 227, 227), feature="squeezenet_v1.1_WithDeepFeature"
         )
 
 
@@ -302,10 +332,20 @@ class ImageSimilarityVisionFeaturePrintSceneTest(ImageSimilarityTest):
     @classmethod
     def setUpClass(self):
         super(ImageSimilarityVisionFeaturePrintSceneTest, self).setUpClass(
-            model="VisionFeaturePrint_Scene", input_image_shape=(3, 299, 299)
+            model="VisionFeaturePrint_Scene", input_image_shape=(3, 299, 299), feature="awesome_image"
         )
 
 
+@unittest.skipIf(
+    _mac_ver() < (10, 14), "VisionFeaturePrint_Scene only supported on macOS 10.14+"
+)
+class ImageSimilarityVisionFeaturePrintSceneTestWithDeepFeatures(ImageSimilarityTest):
+    @classmethod
+    def setUpClass(self):
+        super(ImageSimilarityVisionFeaturePrintSceneTestWithDeepFeatures, self).setUpClass(
+            model="VisionFeaturePrint_Scene", input_image_shape=(3, 299, 299), feature="VisionFeaturePrint_Scene_WithDeepFeature"
+        )
+
 # A test to gaurantee that old code using the incorrect name still works.
 @unittest.skipIf(
     _mac_ver() < (10, 14), "VisionFeaturePrint_Scene only supported on macOS 10.14+"
@@ -314,5 +354,16 @@ class ImageSimilarityVisionFeaturePrintSceneTest_bad_name(ImageSimilarityTest):
     @classmethod
     def setUpClass(self):
         super(ImageSimilarityVisionFeaturePrintSceneTest_bad_name, self).setUpClass(
-            model="VisionFeaturePrint_Screen", input_image_shape=(3, 299, 299)
+            model="VisionFeaturePrint_Screen", input_image_shape=(3, 299, 299), feature="awesome_image"
+        )
+
+
+@unittest.skipIf(
+    _mac_ver() < (10, 14), "VisionFeaturePrint_Scene only supported on macOS 10.14+"
+)
+class ImageSimilarityVisionFeaturePrintSceneTestWithDeepFeatures_bad_name(ImageSimilarityTest):
+    @classmethod
+    def setUpClass(self):
+        super(ImageSimilarityVisionFeaturePrintSceneTestWithDeepFeatures_bad_name, self).setUpClass(
+            model="VisionFeaturePrint_Screen", input_image_shape=(3, 299, 299), feature="VisionFeaturePrint_Scene_WithDeepFeature"
         )

src/python/turicreate/toolkits/image_analysis/__init__.py

@@ -7,6 +7,6 @@
 from __future__ import division as _
 from __future__ import absolute_import as _
 
-__all__ = ["image_analysis"]
+# __all__ = ["image_analysis"]
 
-from . import image_analysis
+from .image_analysis import *