diff --git a/src/python/turicreate/test/test_image_classifier.py b/src/python/turicreate/test/test_image_classifier.py index 3d7d49fd20..a03386b4e1 100644 --- a/src/python/turicreate/test/test_image_classifier.py +++ b/src/python/turicreate/test/test_image_classifier.py @@ -19,6 +19,7 @@ _raise_error_if_not_sframe, _raise_error_if_not_sarray, ) +from turicreate.toolkits.image_analysis.image_analysis import MODEL_TO_FEATURE_SIZE_MAPPING, get_deep_features from . import util as test_util @@ -73,7 +74,10 @@ def get_test_data(): images.append(tc_image) labels = ["white"] * 5 + ["black"] * 5 - return tc.SFrame({"awesome_image": images, "awesome_label": labels}) + data_dict = {"awesome_image": images, "awesome_label": labels} + data = tc.SFrame(data_dict) + + return data data = get_test_data() @@ -84,19 +88,24 @@ class ImageClassifierTest(unittest.TestCase): def setUpClass( self, model="resnet-50", + feature="awesome_image", input_image_shape=(3, 224, 224), tol=0.02, num_examples=100, label_type=int, ): - self.feature = "awesome_image" + self.feature = feature self.target = "awesome_label" self.input_image_shape = input_image_shape self.pre_trained_model = model self.tolerance = tol + # Get deep features if needed + if self.feature.endswith("WithDeepFeature"): + data[self.feature] = get_deep_features(data["awesome_image"], self.feature.split('_WithDeepFeature')[0]) + self.model = tc.image_classifier.create( - data, target=self.target, model=self.pre_trained_model, seed=42 + data, target=self.target, feature=self.feature, model=self.pre_trained_model, seed=42 ) self.nn_model = self.model.feature_extractor self.lm_model = self.model.classifier @@ -132,14 +141,13 @@ def assertListAlmostEquals(self, list1, list2, tol): self.assertAlmostEqual(a, b, delta=tol) def test_create_with_missing_value(self): + data_dict = {} + for col_name, col_type in zip(data.column_names(), data.column_types()): + data_dict[col_name] = tc.SArray([None], dtype=col_type) data_with_none = data.append( - tc.SFrame( - { - self.feature: tc.SArray([None], dtype=tc.Image), - self.target: [data[self.target][0]], - } - ) + tc.SFrame(data_dict) ) + with self.assertRaises(_ToolkitError): tc.image_classifier.create( data_with_none, feature=self.feature, target=self.target @@ -161,6 +169,16 @@ def test_create_with_empty_dataset(self): with self.assertRaises(_ToolkitError): tc.image_classifier.create(data[:0], target=self.target) + def test_select_correct_feature_column_to_train(self): + # sending both, the correct extracted features colum and image column + if self.feature == "awesome_image": + test_data = data.select_columns([self.feature, self.target]) + deep_features_col_name = self.pre_trained_model+"_WithDeepFeature" + test_data[deep_features_col_name] = get_deep_features(data["awesome_image"], + self.pre_trained_model) + test_model = tc.image_classifier.create(test_data, target=self.target, model=self.pre_trained_model) + self.assertTrue(test_model.feature == deep_features_col_name) + def test_predict(self): model = self.model for output_type in ["class", "probability_vector"]: @@ -235,22 +253,26 @@ def test_export_coreml_predict(self): self.model.export_coreml(filename) coreml_model = coremltools.models.MLModel(filename) - img = data[0:1][self.feature][0] - img_fixed = tc.image_analysis.resize(img, *reversed(self.input_image_shape)) - from PIL import Image - - pil_img = Image.fromarray(img_fixed.pixel_data) - - if _mac_ver() >= (10, 13): - classes = self.model.classifier.classes - ret = coreml_model.predict({self.feature: pil_img}) - coreml_values = [ret[self.target + "Probability"][l] for l in classes] - - self.assertListAlmostEquals( - coreml_values, - list(self.model.predict(img_fixed, output_type="probability_vector")), - self.tolerance, - ) + if self.feature == "awesome_image": + img = data[0:1][self.feature][0] + img_fixed = tc.image_analysis.resize(img, *reversed(self.input_image_shape)) + from PIL import Image + + pil_img = Image.fromarray(img_fixed.pixel_data) + + if _mac_ver() >= (10, 13): + classes = self.model.classifier.classes + ret = coreml_model.predict({self.feature: pil_img}) + coreml_values = [ret[self.target + "Probability"][l] for l in classes] + + self.assertListAlmostEquals( + coreml_values, + list(self.model.predict(img_fixed, output_type="probability_vector")), + self.tolerance, + ) + else: + # If the code came here that means the type of the feature used is deep_deatures and the predict fwature in coremltools doesn't work with deep_features yet so we will ignore this specific test case unitl the same is written. + pass def test_classify(self): model = self.model @@ -333,6 +355,18 @@ def test_evaluate_explore(self): evaluation.explore() +class ImageClassifierResnetTestWithDeepFeatures(ImageClassifierTest): + @classmethod + def setUpClass(self): + super(ImageClassifierResnetTestWithDeepFeatures, self).setUpClass( + model="resnet-50", + input_image_shape=(3, 224, 224), + tol=0.02, + num_examples=100, + feature="resnet-50_WithDeepFeature", + ) + + class ImageClassifierSqueezeNetTest(ImageClassifierTest): @classmethod def setUpClass(self): @@ -341,6 +375,18 @@ def setUpClass(self): input_image_shape=(3, 227, 227), tol=0.005, num_examples=200, + feature="awesome_image", + ) + +class ImageClassifierSqueezeNetTestWithDeepFeatures(ImageClassifierTest): + @classmethod + def setUpClass(self): + super(ImageClassifierSqueezeNetTestWithDeepFeatures, self).setUpClass( + model="squeezenet_v1.1", + input_image_shape=(3, 227, 227), + tol=0.005, + num_examples=200, + feature="squeezenet_v1.1_WithDeepFeature", ) @@ -357,4 +403,22 @@ def setUpClass(self): tol=0.005, num_examples=100, label_type=str, + feature="awesome_image", + ) + + +# TODO: if on skip OS, test negative case +@unittest.skipIf( + _mac_ver() < (10, 14), "VisionFeaturePrint_Scene only supported on macOS 10.14+" +) +class VisionFeaturePrintSceneTestWithDeepFeatures(ImageClassifierTest): + @classmethod + def setUpClass(self): + super(VisionFeaturePrintSceneTestWithDeepFeatures, self).setUpClass( + model="VisionFeaturePrint_Scene", + input_image_shape=(3, 299, 299), + tol=0.005, + num_examples=100, + label_type=str, + feature="VisionFeaturePrint_Scene_WithDeepFeature", ) diff --git a/src/python/turicreate/test/test_image_similarity.py b/src/python/turicreate/test/test_image_similarity.py index 21bfae2b48..925723bd45 100644 --- a/src/python/turicreate/test/test_image_similarity.py +++ b/src/python/turicreate/test/test_image_similarity.py @@ -12,8 +12,11 @@ from turicreate.toolkits._internal_utils import _mac_ver import tempfile from . import util as test_util -import numpy as np + from turicreate.toolkits._main import ToolkitError as _ToolkitError +from turicreate.toolkits.image_analysis.image_analysis import MODEL_TO_FEATURE_SIZE_MAPPING, get_deep_features + +import numpy as np def get_test_data(): @@ -62,7 +65,10 @@ def get_test_data(): ) images.append(tc_image) - return tc.SFrame({"awesome_image": images}) + data_dict = {"awesome_image": images} + data = tc.SFrame(data_dict) + + return data data = get_test_data() @@ -70,11 +76,11 @@ def get_test_data(): class ImageSimilarityTest(unittest.TestCase): @classmethod - def setUpClass(self, input_image_shape=(3, 224, 224), model="resnet-50"): + def setUpClass(self, input_image_shape=(3, 224, 224), model="resnet-50", feature="awesome_image"): """ The setup class method for the basic test case with all default values. """ - self.feature = "awesome_image" + self.feature = feature self.label = None self.input_image_shape = input_image_shape self.pre_trained_model = model @@ -85,6 +91,10 @@ def setUpClass(self, input_image_shape=(3, 224, 224), model="resnet-50"): "verbose": True, } + # Get deep features if needed + if self.feature.endswith("WithDeepFeature"): + data[self.feature] = get_deep_features(data["awesome_image"], self.feature.split('_WithDeepFeature')[0]) + # Model self.model = tc.image_similarity.create( data, feature=self.feature, label=None, model=self.pre_trained_model @@ -251,21 +261,25 @@ def get_psnr(x, y): ) # Get model distances for comparison - img = data[0:1][self.feature][0] - img_fixed = tc.image_analysis.resize(img, *reversed(self.input_image_shape)) - tc_ret = self.model.query(img_fixed, k=data.num_rows()) - - if _mac_ver() >= (10, 13): - from PIL import Image as _PIL_Image - - pil_img = _PIL_Image.fromarray(img_fixed.pixel_data) - coreml_ret = coreml_model.predict({"awesome_image": pil_img}) - - # Compare distances - coreml_distances = np.array(coreml_ret["distance"]) - tc_distances = tc_ret.sort("reference_label")["distance"].to_numpy() - psnr_value = get_psnr(coreml_distances, tc_distances) - self.assertTrue(psnr_value > 50) + if self.feature == "awesome_image": + img = data[0:1][self.feature][0] + img_fixed = tc.image_analysis.resize(img, *reversed(self.input_image_shape)) + tc_ret = self.model.query(img_fixed, k=data.num_rows()) + + if _mac_ver() >= (10, 13): + from PIL import Image as _PIL_Image + + pil_img = _PIL_Image.fromarray(img_fixed.pixel_data) + coreml_ret = coreml_model.predict({"awesome_image": pil_img}) + + # Compare distances + coreml_distances = np.array(coreml_ret["distance"]) + tc_distances = tc_ret.sort("reference_label")["distance"].to_numpy() + psnr_value = get_psnr(coreml_distances, tc_distances) + self.assertTrue(psnr_value > 50) + else: + # If the code came here that means the type of the feature used is deep_deatures and the predict fwature in coremltools doesn't work with deep_features yet so we will ignore this specific test case unitl the same is written. + pass def test_save_and_load(self): with test_util.TempDirectory() as filename: @@ -287,11 +301,27 @@ def test_save_and_load(self): print("Export coreml passed") +class ImageSimilarityResnetTestWithDeepFeatures(ImageSimilarityTest): + @classmethod + def setUpClass(self): + super(ImageSimilarityResnetTestWithDeepFeatures, self).setUpClass( + model="resnet-50", input_image_shape=(3, 224, 224), feature="resnet-50_WithDeepFeature" + ) + + class ImageSimilaritySqueezeNetTest(ImageSimilarityTest): @classmethod def setUpClass(self): super(ImageSimilaritySqueezeNetTest, self).setUpClass( - model="squeezenet_v1.1", input_image_shape=(3, 227, 227) + model="squeezenet_v1.1", input_image_shape=(3, 227, 227), feature="awesome_image" + ) + + +class ImageSimilaritySqueezeNetTestWithDeepFeatures(ImageSimilarityTest): + @classmethod + def setUpClass(self): + super(ImageSimilaritySqueezeNetTestWithDeepFeatures, self).setUpClass( + model="squeezenet_v1.1", input_image_shape=(3, 227, 227), feature="squeezenet_v1.1_WithDeepFeature" ) @@ -302,10 +332,20 @@ class ImageSimilarityVisionFeaturePrintSceneTest(ImageSimilarityTest): @classmethod def setUpClass(self): super(ImageSimilarityVisionFeaturePrintSceneTest, self).setUpClass( - model="VisionFeaturePrint_Scene", input_image_shape=(3, 299, 299) + model="VisionFeaturePrint_Scene", input_image_shape=(3, 299, 299), feature="awesome_image" ) +@unittest.skipIf( + _mac_ver() < (10, 14), "VisionFeaturePrint_Scene only supported on macOS 10.14+" +) +class ImageSimilarityVisionFeaturePrintSceneTestWithDeepFeatures(ImageSimilarityTest): + @classmethod + def setUpClass(self): + super(ImageSimilarityVisionFeaturePrintSceneTestWithDeepFeatures, self).setUpClass( + model="VisionFeaturePrint_Scene", input_image_shape=(3, 299, 299), feature="VisionFeaturePrint_Scene_WithDeepFeature" + ) + # A test to gaurantee that old code using the incorrect name still works. @unittest.skipIf( _mac_ver() < (10, 14), "VisionFeaturePrint_Scene only supported on macOS 10.14+" @@ -314,5 +354,16 @@ class ImageSimilarityVisionFeaturePrintSceneTest_bad_name(ImageSimilarityTest): @classmethod def setUpClass(self): super(ImageSimilarityVisionFeaturePrintSceneTest_bad_name, self).setUpClass( - model="VisionFeaturePrint_Screen", input_image_shape=(3, 299, 299) + model="VisionFeaturePrint_Screen", input_image_shape=(3, 299, 299), feature="awesome_image" + ) + + +@unittest.skipIf( + _mac_ver() < (10, 14), "VisionFeaturePrint_Scene only supported on macOS 10.14+" +) +class ImageSimilarityVisionFeaturePrintSceneTestWithDeepFeatures_bad_name(ImageSimilarityTest): + @classmethod + def setUpClass(self): + super(ImageSimilarityVisionFeaturePrintSceneTestWithDeepFeatures_bad_name, self).setUpClass( + model="VisionFeaturePrint_Screen", input_image_shape=(3, 299, 299), feature="VisionFeaturePrint_Scene_WithDeepFeature" ) diff --git a/src/python/turicreate/toolkits/image_analysis/__init__.py b/src/python/turicreate/toolkits/image_analysis/__init__.py index 4b31b6b920..fe9c8f5d2e 100644 --- a/src/python/turicreate/toolkits/image_analysis/__init__.py +++ b/src/python/turicreate/toolkits/image_analysis/__init__.py @@ -7,6 +7,6 @@ from __future__ import division as _ from __future__ import absolute_import as _ -__all__ = ["image_analysis"] +# __all__ = ["image_analysis"] -from . import image_analysis +from .image_analysis import * \ No newline at end of file diff --git a/src/python/turicreate/toolkits/image_analysis/image_analysis.py b/src/python/turicreate/toolkits/image_analysis/image_analysis.py index 2ad21c5ec6..705cdde023 100644 --- a/src/python/turicreate/toolkits/image_analysis/image_analysis.py +++ b/src/python/turicreate/toolkits/image_analysis/image_analysis.py @@ -6,8 +6,22 @@ from __future__ import print_function as _ from __future__ import division as _ from __future__ import absolute_import as _ + +import turicreate as _tc + +import turicreate.toolkits._internal_utils as _tkutl +from turicreate.toolkits._main import ToolkitError as _ToolkitError +import turicreate.toolkits._internal_utils as _tkutl +from .._internal_utils import _mac_ver +from .. import _pre_trained_models from ...data_structures.image import Image as _Image +from .. import _image_feature_extractor +MODEL_TO_FEATURE_SIZE_MAPPING = { + "resnet-50": 2048, + "squeezenet_v1.1": 1000, + "VisionFeaturePrint_Scene": 2048 +} def load_images( url, @@ -177,3 +191,101 @@ def resize(image, width, height, channels=None, decode=False, resample="nearest" raise ValueError( "Cannot call 'resize' on objects that are not either an Image or SArray of Images" ) + + +def get_deep_features(images, model_name, batch_size=64, verbose=True): + """ + Extracts features from images from a specific model. + + Parameters + ---------- + images : SArray + Input data. + + model_name : string + string optional + Uses a pretrained model to bootstrap an image classifier: + + - "resnet-50" : Uses a pretrained resnet model. + Exported Core ML model will be ~90M. + + - "squeezenet_v1.1" : Uses a pretrained squeezenet model. + Exported Core ML model will be ~4.7M. + + - "VisionFeaturePrint_Scene": Uses an OS internal feature extractor. + Only on available on iOS 12.0+, + macOS 10.14+ and tvOS 12.0+. + Exported Core ML model will be ~41K. + + Models are downloaded from the internet if not available locally. Once + downloaded, the models are cached for future use. + + Returns + ------- + out : SArray + Returns an SArray with all the extracted features. + + Examples + -------- + # Get Deep featuers from an sarray of images + >>> url ='https://static.turi.com/datasets/images/nested' + >>> image_sframe = turicreate.image_analysis.load_images(url, "auto", with_path=False, recursive=True) + >>> image_sarray = image_sframe["image"] + >>> deep_features_sframe = turicreate.image_analysis.get_deep_features(image_sarray, model_name="resnet-50") + + """ + + # Check model parameter + allowed_models = list(_pre_trained_models.IMAGE_MODELS.keys()) + if _mac_ver() >= (10, 14): + allowed_models.append("VisionFeaturePrint_Scene") + _tkutl._check_categorical_option_type("model", model_name, allowed_models) + + # Check images parameter + if not isinstance(images, _tc.SArray): + raise TypeError("Unrecognized type for 'images'. An SArray is expected.") + if len(images) == 0: + raise _ToolkitError("Unable to extract features on an empty SArray object") + + if batch_size < 1: + raise ValueError("'batch_size' must be greater than or equal to 1") + + # Extract features + feature_extractor = _image_feature_extractor._create_feature_extractor(model_name) + images_sf = _tc.SFrame({"image":images}) + return feature_extractor.extract_features(images_sf, "image", verbose=verbose, + batch_size=batch_size) + + +def _find_only_image_extracted_features_column(sframe, model_name): + """ + Finds the only column in `sframe` with a type of array.array and has + the length same as the last layer of the model in use. + If there are zero or more than one image columns, an exception will + be raised. + """ + from array import array + + feature_column = _tkutl._find_only_column_of_type(sframe, target_type=array, type_name="array", col_name="deep_features") + if _is_image_deep_feature_sarray(sframe[feature_column], model_name): + return feature_column + else: + raise _ToolkitError('No "{col_name}" column specified and no column with expected type "{type_name}" is found.'.format(col_name="deep_features", type_name="array") + ) + + +def _is_image_deep_feature_sarray(feature_sarray, model_name): + """ + Finds if the given `SArray` has extracted features for a given model_name. + """ + from array import array + + if not (len(feature_sarray) > 0): + return False + if feature_sarray.dtype != array: + return False + if type(feature_sarray[0]) != array: + return False + if len(feature_sarray[0]) != MODEL_TO_FEATURE_SIZE_MAPPING[model_name]: + return False + return True diff --git a/src/python/turicreate/toolkits/image_classifier/image_classifier.py b/src/python/turicreate/toolkits/image_classifier/image_classifier.py index e2009ad963..bea93b0ca3 100644 --- a/src/python/turicreate/toolkits/image_classifier/image_classifier.py +++ b/src/python/turicreate/toolkits/image_classifier/image_classifier.py @@ -22,6 +22,7 @@ from .._internal_utils import _mac_ver from .. import _pre_trained_models from .. import _image_feature_extractor +from ..image_analysis import image_analysis from ._evaluation import Evaluation as _Evaluation _DEFAULT_SOLVER_OPTIONS = { @@ -71,10 +72,10 @@ def create( the order in which the classes are mapped. feature : string, optional - indicates that the SFrame has only column of Image type and that will - Name of the column containing the input images. 'None' (the default) - indicates the only image column in `dataset` should be used as the - feature. + indicates that the SFrame has either column of Image type or array type + (extracted features) and that will be the name of the column containing the input + images or features. 'None' (the default) indicates that only feature column or the + only image column in `dataset` should be used as the feature. l2_penalty : float, optional Weight on l2 regularization of the model. The larger this weight, the @@ -251,29 +252,66 @@ def create( raise TypeError("Unrecognized value for 'validation_set'.") if feature is None: - feature = _tkutl._find_only_image_column(dataset) + try: + feature = image_analysis._find_only_image_extracted_features_column(dataset, model) + feature_type = "extracted_features_array" + except: + feature = None + + if feature is None: + feature = _tkutl._find_only_image_column(dataset) + feature_type = "image" + else: + if image_analysis._is_image_deep_feature_sarray(dataset[feature], model): + feature_type = "extracted_features_array" + elif dataset[feature].dtype is _tc.Image: + feature_type = "image" + else: + raise _ToolkitError('The "{feature}" column of the sFrame neither has the dataype image or extracted features array.'.format(feature=feature) + + ' "Datasets" consists of columns with types: ' + + ", ".join([x.__name__ for x in dataset.column_types()]) + + "." + ) + _tkutl._handle_missing_values(dataset, feature, "training_dataset") feature_extractor = _image_feature_extractor._create_feature_extractor(model) - - # Extract features - extracted_features = _tc.SFrame( - { - target: dataset[target], - "__image_features__": feature_extractor.extract_features( - dataset, feature, verbose=verbose, batch_size=batch_size - ), - } - ) - if isinstance(validation_set, _tc.SFrame): - _tkutl._handle_missing_values(dataset, feature, "validation_set") - extracted_features_validation = _tc.SFrame( + if feature_type == "image": + # Extract features + extracted_features = _tc.SFrame( { - target: validation_set[target], + target: dataset[target], "__image_features__": feature_extractor.extract_features( - validation_set, feature, verbose=verbose, batch_size=batch_size + dataset, feature, verbose=verbose, batch_size=batch_size ), } ) + else: + extracted_features = _tc.SFrame( + { + target: dataset[target], + "__image_features__": dataset[feature] + } + ) + + # Validation set + if isinstance(validation_set, _tc.SFrame): + if feature_type == "image": + _tkutl._handle_missing_values(validation_set, feature, "validation_set") + extracted_features_validation = _tc.SFrame( + { + target: validation_set[target], + "__image_features__": feature_extractor.extract_features( + validation_set, feature, verbose=verbose, batch_size=batch_size + ), + } + ) + else: + extracted_features_validation = _tc.SFrame( + { + target: validation_set[target], + "__image_features__": validation_set[feature] + } + ) else: extracted_features_validation = validation_set @@ -444,12 +482,15 @@ def _canonize_input(self, dataset): along with an unpack callback function that can be applied to prediction results to "undo" the canonization. """ + from array import array + unpack = lambda x: x if isinstance(dataset, _tc.SArray): dataset = _tc.SFrame({self.feature: dataset}) - elif isinstance(dataset, _tc.Image): + elif isinstance(dataset, (_tc.Image, array)): dataset = _tc.SFrame({self.feature: [dataset]}) unpack = lambda x: x[0] + return dataset, unpack def predict(self, dataset, output_type="class", batch_size=64): @@ -471,8 +512,8 @@ def predict(self, dataset, output_type="class", batch_size=64): Parameters ---------- - dataset : SFrame | SArray | turicreate.Image - The images to be classified. + dataset : SFrame | SArray | turicreate.Image | array + The images to be classified or extracted features. If dataset is an SFrame, it must have columns with the same names as the features used for model training, but does not require a target column. Additional columns are ignored. @@ -510,7 +551,9 @@ class as a vector. The probability of the first class (sorted >>> class_predictions = model.predict(data, output_type='class') """ - if not isinstance(dataset, (_tc.SFrame, _tc.SArray, _tc.Image)): + from array import array + + if not isinstance(dataset, (_tc.SFrame, _tc.SArray, _tc.Image, array)): raise TypeError( "dataset must be either an SFrame, SArray or turicreate.Image" ) @@ -559,7 +602,9 @@ def classify(self, dataset, batch_size=64): >>> classes = model.classify(data) """ - if not isinstance(dataset, (_tc.SFrame, _tc.SArray, _tc.Image)): + from array import array + + if not isinstance(dataset, (_tc.SFrame, _tc.SArray, _tc.Image, array)): raise TypeError( "dataset must be either an SFrame, SArray or turicreate.Image" ) @@ -626,7 +671,9 @@ def predict_topk(self, dataset, output_type="probability", k=3, batch_size=64): +----+-------+-------------------+ [35688 rows x 3 columns] """ - if not isinstance(dataset, (_tc.SFrame, _tc.SArray, _tc.Image)): + from array import array + + if not isinstance(dataset, (_tc.SFrame, _tc.SArray, _tc.Image, array)): raise TypeError( "dataset must be either an SFrame, SArray or turicreate.Image" ) @@ -836,13 +883,26 @@ def evaluate(self, dataset, metric="auto", verbose=True, batch_size=64): return _Evaluation(evaluation_result) def _extract_features(self, dataset, verbose=False, batch_size=64): - return _tc.SFrame( - { - "__image_features__": self.feature_extractor.extract_features( - dataset, self.feature, verbose=verbose, batch_size=batch_size - ) - } - ) + if image_analysis._is_image_deep_feature_sarray(dataset[self.feature], self.model): + return _tc.SFrame( + { + "__image_features__": dataset[self.feature] + } + ) + elif dataset[self.feature].dtype is _tc.Image: + return _tc.SFrame( + { + "__image_features__": self.feature_extractor.extract_features( + dataset, self.feature, verbose=verbose, batch_size=batch_size + ) + } + ) + else: + raise _ToolkitError('The "{feature}" column of the SFrame neither has the dataype image or extracted features array.'.format(feature=feature) + + ' "Datasets" consists of columns with types: ' + + ", ".join([x.__name__ for x in dataset.column_types()]) + + "." + ) def export_coreml(self, filename): """ diff --git a/src/python/turicreate/toolkits/image_similarity/image_similarity.py b/src/python/turicreate/toolkits/image_similarity/image_similarity.py index bab3017153..8a1ec92043 100644 --- a/src/python/turicreate/toolkits/image_similarity/image_similarity.py +++ b/src/python/turicreate/toolkits/image_similarity/image_similarity.py @@ -19,6 +19,7 @@ from .._internal_utils import _mac_ver from .. import _pre_trained_models from .. import _image_feature_extractor +from ..image_analysis import image_analysis def create( @@ -39,9 +40,9 @@ def create( identify reference dataset rows when the model is queried. feature : string - Name of the column containing the input images. 'None' (the default) - indicates that the SFrame has only one column of Image type and that will - be used for similarity. + Name of the column containing either the input images or extracted features. + 'None' (the default) indicates that only feature column or the only image + column in `dataset` should be used as the feature. model: string, optional Uses a pretrained model to bootstrap an image similarity model @@ -127,18 +128,52 @@ def create( # Set defaults if feature is None: - feature = _tkutl._find_only_image_column(dataset) + # select feature column : either extracted features columns or image column itself + try: + feature = image_analysis._find_only_image_extracted_features_column(dataset, model) + feature_type = "extracted_features_array" + except: + feature = None + if feature is None: + try: + feature = _tkutl._find_only_image_column(dataset) + feature_type = "image" + except: + raise _ToolkitError( + 'No feature column specified and no column with expected type image or array is found.' + + ' "datasets" consists of columns with types: ' + + ", ".join([x.__name__ for x in dataset.column_types()]) + + "." + ) + else: + if image_analysis._is_image_deep_feature_sarray(dataset[feature], model): + feature_type = "extracted_features_array" + elif dataset[feature].dtype is _tc.Image: + feature_type = "image" + else: + raise _ToolkitError('The "{feature}" column of the sFrame neither has the dataype image or array (for extracted features)'.format(feature=feature) + + ' "datasets" consists of columns with types: ' + + ", ".join([x.__name__ for x in dataset.column_types()]) + + "." + ) + _tkutl._handle_missing_values(dataset, feature) feature_extractor = _image_feature_extractor._create_feature_extractor(model) - - # Extract features - extracted_features = _tc.SFrame( - { - "__image_features__": feature_extractor.extract_features( - dataset, feature, verbose=verbose, batch_size=batch_size - ), - } - ) + if feature_type == "image": + # Extract features + extracted_features = _tc.SFrame( + { + "__image_features__": feature_extractor.extract_features( + dataset, feature, verbose=verbose, batch_size=batch_size + ), + } + ) + else: + extracted_features = _tc.SFrame( + { + "__image_features__": dataset[feature] + } + ) # Train a similarity model using the extracted features if label is not None: @@ -299,14 +334,27 @@ def _get_summary_struct(self): section_titles = ["Schema", "Training summary"] return ([model_fields, training_fields], section_titles) - def _extract_features(self, dataset, verbose, batch_size=64): - return _tc.SFrame( - { - "__image_features__": self.feature_extractor.extract_features( - dataset, self.feature, verbose=verbose, batch_size=batch_size - ) - } - ) + def _extract_features(self, dataset, verbose=False, batch_size=64): + if image_analysis._is_image_deep_feature_sarray(dataset[self.feature], self.model): + return _tc.SFrame( + { + "__image_features__": dataset[self.feature] + } + ) + elif dataset[self.feature].dtype is _tc.Image: + return _tc.SFrame( + { + "__image_features__": self.feature_extractor.extract_features( + dataset, self.feature, verbose=verbose, batch_size=batch_size + ) + } + ) + else: + raise _ToolkitError('The "{feature}" column of the sFrame neither has the dataype image or extracted features array.'.format(feature=feature) + + ' "Datasets" consists of columns with types: ' + + ", ".join([x.__name__ for x in dataset.column_types()]) + + "." + ) def query(self, dataset, label=None, k=5, radius=None, verbose=True, batch_size=64): """ @@ -381,7 +429,9 @@ def query(self, dataset, label=None, k=5, radius=None, verbose=True, batch_size= | 2 | 1 | 0.464004310325 | 2 | +-------------+-----------------+----------------+------+ """ - if not isinstance(dataset, (_tc.SFrame, _tc.SArray, _tc.Image)): + from array import array + + if not isinstance(dataset, (_tc.SFrame, _tc.SArray, _tc.Image, array)): raise TypeError( "dataset must be either an SFrame, SArray or turicreate.Image" ) @@ -390,7 +440,7 @@ def query(self, dataset, label=None, k=5, radius=None, verbose=True, batch_size= if isinstance(dataset, _tc.SArray): dataset = _tc.SFrame({self.feature: dataset}) - elif isinstance(dataset, _tc.Image): + elif isinstance(dataset, (_tc.Image, array)): dataset = _tc.SFrame({self.feature: [dataset]}) extracted_features = self._extract_features(