diff --git a/.gitignore b/.gitignore index 5cfb9504ab1d..625ff5896c3b 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,7 @@ GeneratedPluginRegistrant.m GeneratedPluginRegistrant.java build/ .flutter-plugins + +.project +.classpath +.settings diff --git a/packages/firebase_ml_vision/CHANGELOG.md b/packages/firebase_ml_vision/CHANGELOG.md index 10cbf8ea279f..b2fe1d37d699 100644 --- a/packages/firebase_ml_vision/CHANGELOG.md +++ b/packages/firebase_ml_vision/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.9.2 + +* Add detection of `FaceContour`s when using the `FaceDetector`. See `README.md` for more information. + ## 0.9.1+1 * Update google-services Android gradle plugin to 4.3.0 in documentation and examples. diff --git a/packages/firebase_ml_vision/README.md b/packages/firebase_ml_vision/README.md index 5e6b9dbd38d7..be7e6578f3ab 100644 --- a/packages/firebase_ml_vision/README.md +++ b/packages/firebase_ml_vision/README.md @@ -25,6 +25,18 @@ android { } ``` +If you're using the on-device `Face Contour Detection`, include the latest matching [ML Kit: Face Detection Model](https://firebase.google.com/support/release-notes/android) dependency in your app-level build.gradle file. + +``` +android { + dependencies { + // ... + + api 'com.google.firebase:firebase-ml-vision-face-model:17.0.2' + } +} +``` + If you receive compilation errors, try an earlier version of [ML Kit: Image Labeling](https://firebase.google.com/support/release-notes/android). Optional but recommended: If you use the on-device API, configure your app to automatically download the ML model to the device after your app is installed from the Play Store. To do so, add the following declaration to your app's AndroidManifest.xml file: diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FaceDetector.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FaceDetector.java index 245c2109074a..eba06e71eecd 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FaceDetector.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/FaceDetector.java @@ -5,7 +5,9 @@ import com.google.android.gms.tasks.OnSuccessListener; import com.google.firebase.ml.vision.FirebaseVision; import com.google.firebase.ml.vision.common.FirebaseVisionImage; +import com.google.firebase.ml.vision.common.FirebaseVisionPoint; import com.google.firebase.ml.vision.face.FirebaseVisionFace; +import com.google.firebase.ml.vision.face.FirebaseVisionFaceContour; import com.google.firebase.ml.vision.face.FirebaseVisionFaceDetector; import com.google.firebase.ml.vision.face.FirebaseVisionFaceDetectorOptions; import com.google.firebase.ml.vision.face.FirebaseVisionFaceLandmark; @@ -63,6 +65,8 @@ public void onSuccess(List firebaseVisionFaces) { faceData.put("landmarks", getLandmarkData(face)); + faceData.put("contours", getContourData(face)); + faces.add(faceData); } @@ -95,6 +99,34 @@ private Map getLandmarkData(FirebaseVisionFace face) { return landmarks; } + private Map> getContourData(FirebaseVisionFace face) { + Map> contours = new HashMap<>(); + + contours.put("allPoints", contourPosition(face, FirebaseVisionFaceContour.ALL_POINTS)); + contours.put("face", contourPosition(face, FirebaseVisionFaceContour.FACE)); + contours.put("leftEye", contourPosition(face, FirebaseVisionFaceContour.LEFT_EYE)); + contours.put( + "leftEyebrowBottom", contourPosition(face, FirebaseVisionFaceContour.LEFT_EYEBROW_BOTTOM)); + contours.put( + "leftEyebrowTop", contourPosition(face, FirebaseVisionFaceContour.LEFT_EYEBROW_TOP)); + contours.put( + "lowerLipBottom", contourPosition(face, FirebaseVisionFaceContour.LOWER_LIP_BOTTOM)); + contours.put("lowerLipTop", contourPosition(face, FirebaseVisionFaceContour.LOWER_LIP_TOP)); + contours.put("noseBottom", contourPosition(face, FirebaseVisionFaceContour.NOSE_BOTTOM)); + contours.put("noseBridge", contourPosition(face, FirebaseVisionFaceContour.NOSE_BRIDGE)); + contours.put("rightEye", contourPosition(face, FirebaseVisionFaceContour.RIGHT_EYE)); + contours.put( + "rightEyebrowBottom", + contourPosition(face, FirebaseVisionFaceContour.RIGHT_EYEBROW_BOTTOM)); + contours.put( + "rightEyebrowTop", contourPosition(face, FirebaseVisionFaceContour.RIGHT_EYEBROW_TOP)); + contours.put( + "upperLipBottom", contourPosition(face, FirebaseVisionFaceContour.UPPER_LIP_BOTTOM)); + contours.put("upperLipTop", contourPosition(face, FirebaseVisionFaceContour.UPPER_LIP_TOP)); + + return contours; + } + private double[] landmarkPosition(FirebaseVisionFace face, int landmarkInt) { FirebaseVisionFaceLandmark landmark = face.getLandmark(landmarkInt); if (landmark != null) { @@ -104,6 +136,22 @@ private double[] landmarkPosition(FirebaseVisionFace face, int landmarkInt) { return null; } + private List contourPosition(FirebaseVisionFace face, int contourInt) { + FirebaseVisionFaceContour contour = face.getContour(contourInt); + if (contour != null) { + List contourPoints = contour.getPoints(); + List result = new ArrayList(); + + for (int i = 0; i < contourPoints.size(); i++) { + result.add(new double[] {contourPoints.get(i).getX(), contourPoints.get(i).getY()}); + } + + return result; + } + + return null; + } + private FirebaseVisionFaceDetectorOptions parseOptions(Map options) { int classification = (boolean) options.get("enableClassification") @@ -115,6 +163,11 @@ private FirebaseVisionFaceDetectorOptions parseOptions(Map optio ? FirebaseVisionFaceDetectorOptions.ALL_LANDMARKS : FirebaseVisionFaceDetectorOptions.NO_LANDMARKS; + int contours = + (boolean) options.get("enableContours") + ? FirebaseVisionFaceDetectorOptions.ALL_CONTOURS + : FirebaseVisionFaceDetectorOptions.NO_CONTOURS; + int mode; switch ((String) options.get("mode")) { case "accurate": @@ -131,6 +184,7 @@ private FirebaseVisionFaceDetectorOptions parseOptions(Map optio new FirebaseVisionFaceDetectorOptions.Builder() .setClassificationMode(classification) .setLandmarkMode(landmark) + .setContourMode(contours) .setMinFaceSize((float) ((double) options.get("minFaceSize"))) .setPerformanceMode(mode); diff --git a/packages/firebase_ml_vision/example/android/app/build.gradle b/packages/firebase_ml_vision/example/android/app/build.gradle index 96c685ea27a2..edd72b27f848 100644 --- a/packages/firebase_ml_vision/example/android/app/build.gradle +++ b/packages/firebase_ml_vision/example/android/app/build.gradle @@ -48,6 +48,7 @@ android { dependencies { api 'com.google.firebase:firebase-ml-vision-image-label-model:17.0.2' + api 'com.google.firebase:firebase-ml-vision-face-model:17.0.2' } } diff --git a/packages/firebase_ml_vision/example/assets/test_face.jpg b/packages/firebase_ml_vision/example/assets/test_face.jpg index f87458b4db81..a384f3c243b6 100644 Binary files a/packages/firebase_ml_vision/example/assets/test_face.jpg and b/packages/firebase_ml_vision/example/assets/test_face.jpg differ diff --git a/packages/firebase_ml_vision/example/test_driver/face_detector.dart b/packages/firebase_ml_vision/example/test_driver/face_detector.dart index 1f94b4061cae..666b200e301c 100644 --- a/packages/firebase_ml_vision/example/test_driver/face_detector.dart +++ b/packages/firebase_ml_vision/example/test_driver/face_detector.dart @@ -6,7 +6,10 @@ part of 'firebase_ml_vision.dart'; void faceDetectorTests() { group('$FaceDetector', () { - final FaceDetector detector = FirebaseVision.instance.faceDetector(); + final FaceDetector detector = FirebaseVision.instance.faceDetector( + FaceDetectorOptions( + enableContours: true, mode: FaceDetectorMode.accurate), + ); test('processImage', () async { final String tmpFilename = await _loadImage('assets/test_face.jpg'); @@ -16,6 +19,10 @@ void faceDetectorTests() { final List faces = await detector.processImage(visionImage); expect(faces.length, 1); + expect( + faces[0].getContour(FaceContourType.allPoints).positionsList, + isNotEmpty, + ); }); test('close', () { diff --git a/packages/firebase_ml_vision/ios/Classes/FaceDetector.m b/packages/firebase_ml_vision/ios/Classes/FaceDetector.m index 75122b244b77..3f036b403857 100644 --- a/packages/firebase_ml_vision/ios/Classes/FaceDetector.m +++ b/packages/firebase_ml_vision/ios/Classes/FaceDetector.m @@ -68,6 +68,35 @@ - (void)handleDetection:(FIRVisionImage *)image result:(FlutterResult)result { @"rightMouth" : [FaceDetector getLandmarkPosition:face landmark:FIRFaceLandmarkTypeMouthRight], }, + @"contours" : @{ + @"allPoints" : [FaceDetector getContourPoints:face contour:FIRFaceContourTypeAll], + @"face" : [FaceDetector getContourPoints:face contour:FIRFaceContourTypeFace], + @"leftEye" : [FaceDetector getContourPoints:face contour:FIRFaceContourTypeLeftEye], + @"leftEyebrowBottom" : + [FaceDetector getContourPoints:face + contour:FIRFaceContourTypeLeftEyebrowBottom], + @"leftEyebrowTop" : + [FaceDetector getContourPoints:face contour:FIRFaceContourTypeLeftEyebrowTop], + @"lowerLipBottom" : + [FaceDetector getContourPoints:face contour:FIRFaceContourTypeLowerLipBottom], + @"lowerLipTop" : [FaceDetector getContourPoints:face + contour:FIRFaceContourTypeLowerLipTop], + @"noseBottom" : [FaceDetector getContourPoints:face + contour:FIRFaceContourTypeNoseBottom], + @"noseBridge" : [FaceDetector getContourPoints:face + contour:FIRFaceContourTypeNoseBridge], + @"rightEye" : [FaceDetector getContourPoints:face + contour:FIRFaceContourTypeRightEye], + @"rightEyebrowBottom" : + [FaceDetector getContourPoints:face + contour:FIRFaceContourTypeRightEyebrowBottom], + @"rightEyebrowTop" : + [FaceDetector getContourPoints:face contour:FIRFaceContourTypeRightEyebrowTop], + @"upperLipBottom" : + [FaceDetector getContourPoints:face contour:FIRFaceContourTypeUpperLipBottom], + @"upperLipTop" : [FaceDetector getContourPoints:face + contour:FIRFaceContourTypeUpperLipTop], + } }; [faceData addObject:data]; @@ -86,6 +115,21 @@ + (id)getLandmarkPosition:(FIRVisionFace *)face landmark:(FIRFaceLandmarkType)la return [NSNull null]; } ++ (id)getContourPoints:(FIRVisionFace *)face contour:(FIRFaceContourType)contourType { + FIRVisionFaceContour *contour = [face contourOfType:contourType]; + if (contour) { + NSArray *contourPoints = contour.points; + NSMutableArray *result = [[NSMutableArray alloc] initWithCapacity:[contourPoints count]]; + for (int i = 0; i < [contourPoints count]; i++) { + FIRVisionPoint *point = [contourPoints objectAtIndex:i]; + [result insertObject:@[ point.x, point.y ] atIndex:i]; + } + return [result copy]; + } + + return [NSNull null]; +} + + (FIRVisionFaceDetectorOptions *)parseOptions:(NSDictionary *)optionsData { FIRVisionFaceDetectorOptions *options = [[FIRVisionFaceDetectorOptions alloc] init]; @@ -103,6 +147,13 @@ + (FIRVisionFaceDetectorOptions *)parseOptions:(NSDictionary *)optionsData { options.landmarkMode = FIRVisionFaceDetectorLandmarkModeNone; } + NSNumber *enableContours = optionsData[@"enableContours"]; + if (enableContours.boolValue) { + options.contourMode = FIRVisionFaceDetectorContourModeAll; + } else { + options.contourMode = FIRVisionFaceDetectorContourModeNone; + } + NSNumber *enableTracking = optionsData[@"enableTracking"]; options.trackingEnabled = enableTracking.boolValue; diff --git a/packages/firebase_ml_vision/lib/src/face_detector.dart b/packages/firebase_ml_vision/lib/src/face_detector.dart index 4ada4ea3c677..aa4d0d76f560 100644 --- a/packages/firebase_ml_vision/lib/src/face_detector.dart +++ b/packages/firebase_ml_vision/lib/src/face_detector.dart @@ -24,6 +24,24 @@ enum FaceLandmarkType { rightMouth, } +/// Available face contour types detected by [FaceDetector]. +enum FaceContourType { + allPoints, + face, + leftEye, + leftEyebrowBottom, + leftEyebrowTop, + lowerLipBottom, + lowerLipTop, + noseBottom, + noseBridge, + rightEye, + rightEyebrowBottom, + rightEyebrowTop, + upperLipBottom, + upperLipTop +} + /// Detector for detecting faces in an input image. /// /// A face detector is created via @@ -59,6 +77,7 @@ class FaceDetector { 'options': { 'enableClassification': options.enableClassification, 'enableLandmarks': options.enableLandmarks, + 'enableContours': options.enableContours, 'enableTracking': options.enableTracking, 'minFaceSize': options.minFaceSize, 'mode': _enumToString(options.mode), @@ -98,6 +117,7 @@ class FaceDetectorOptions { const FaceDetectorOptions({ this.enableClassification = false, this.enableLandmarks = false, + this.enableContours = false, this.enableTracking = false, this.minFaceSize = 0.1, this.mode = FaceDetectorMode.fast, @@ -112,6 +132,9 @@ class FaceDetectorOptions { /// Whether to detect [FaceLandmark]s. final bool enableLandmarks; + /// Whether to detect [FaceContour]s. + final bool enableContours; + /// Whether to enable face tracking. /// /// If enabled, the detector will maintain a consistent ID for each face when @@ -154,9 +177,25 @@ class Face { type, Offset(pos[0], pos[1]), ); + })), + _contours = Map.fromIterables( + FaceContourType.values, + FaceContourType.values.map((FaceContourType type) { + /// added empty map to pass the tests + final List arr = + (data['contours'] ?? {})[_enumToString(type)]; + return (arr == null) + ? null + : FaceContour._( + type, + arr + .map((dynamic pos) => Offset(pos[0], pos[1])) + .toList(), + ); })); final Map _landmarks; + final Map _contours; /// The axis-aligned bounding rectangle of the detected face. /// @@ -212,6 +251,11 @@ class Face { /// /// Null if landmark was not detected. FaceLandmark getLandmark(FaceLandmarkType landmark) => _landmarks[landmark]; + + /// Gets the contour based on the provided [FaceContourType]. + /// + /// Null if contour was not detected. + FaceContour getContour(FaceContourType contour) => _contours[contour]; } /// Represent a face landmark. @@ -228,3 +272,18 @@ class FaceLandmark { /// The point (0, 0) is defined as the upper-left corner of the image. final Offset position; } + +/// Represent a face contour. +/// +/// Contours of facial features. +class FaceContour { + FaceContour._(this.type, this.positionsList); + + /// The [FaceContourType] of this contour. + final FaceContourType type; + + /// Gets a 2D point [List] for contour positions. + /// + /// The point (0, 0) is defined as the upper-left corner of the image. + final List positionsList; +} diff --git a/packages/firebase_ml_vision/pubspec.yaml b/packages/firebase_ml_vision/pubspec.yaml index abc6566e8e8f..05bdfe95d299 100644 --- a/packages/firebase_ml_vision/pubspec.yaml +++ b/packages/firebase_ml_vision/pubspec.yaml @@ -2,7 +2,7 @@ name: firebase_ml_vision description: Flutter plugin for Firebase machine learning vision services. author: Flutter Team homepage: https://github.com/flutter/plugins/tree/master/packages/firebase_ml_vision -version: 0.9.1+1 +version: 0.9.2 dependencies: flutter: diff --git a/packages/firebase_ml_vision/test/firebase_ml_vision_test.dart b/packages/firebase_ml_vision/test/firebase_ml_vision_test.dart index 85e069774b9a..59dea115f83f 100644 --- a/packages/firebase_ml_vision/test/firebase_ml_vision_test.dart +++ b/packages/firebase_ml_vision/test/firebase_ml_vision_test.dart @@ -526,6 +526,64 @@ void main() { 'rightEye': [16.1, 17.1], 'rightMouth': [18.1, 19.1], }, + 'contours': { + 'allPoints': [ + [1.1, 2.2], + [3.3, 4.4], + ], + 'face': [ + [1.1, 2.2], + [3.3, 4.4], + ], + 'leftEye': [ + [1.1, 2.2], + [3.3, 4.4], + ], + 'leftEyebrowBottom': [ + [1.1, 2.2], + [3.3, 4.4], + ], + 'leftEyebrowTop': [ + [1.1, 2.2], + [3.3, 4.4], + ], + 'lowerLipBottom': [ + [1.1, 2.2], + [3.3, 4.4], + ], + 'lowerLipTop': [ + [1.1, 2.2], + [3.3, 4.4], + ], + 'noseBottom': [ + [1.1, 2.2], + [3.3, 4.4], + ], + 'noseBridge': [ + [1.1, 2.2], + [3.3, 4.4], + ], + 'rightEye': [ + [1.1, 2.2], + [3.3, 4.4], + ], + 'rightEyebrowBottom': [ + [1.1, 2.2], + [3.3, 4.4], + ], + 'rightEyebrowTop': [ + [1.1, 2.2], + [3.3, 4.4], + ], + 'upperLipBottom': [ + [1.1, 2.2], + [3.3, 4.4], + ], + 'upperLipTop': [ + [1.1, 2.2], + [3.3, 4.4], + ], + }, }, ]; }); @@ -538,6 +596,7 @@ void main() { enableClassification: true, enableLandmarks: true, enableTracking: false, + enableContours: true, minFaceSize: 0.5, mode: FaceDetectorMode.accurate, ), @@ -561,6 +620,7 @@ void main() { 'options': { 'enableClassification': true, 'enableLandmarks': true, + 'enableContours': true, 'enableTracking': false, 'minFaceSize': 0.5, 'mode': 'accurate', @@ -598,6 +658,81 @@ void main() { expect(p(FaceLandmarkType.rightEar), const Offset(14.1, 15.1)); expect(p(FaceLandmarkType.rightEye), const Offset(16.1, 17.1)); expect(p(FaceLandmarkType.rightMouth), const Offset(18.1, 19.1)); + + List c(FaceContourType type) { + return face.getContour(type).positionsList; + } + + expect( + c(FaceContourType.allPoints), + containsAllInOrder( + [const Offset(1.1, 2.2), const Offset(3.3, 4.4)]), + ); + expect( + c(FaceContourType.face), + containsAllInOrder( + [const Offset(1.1, 2.2), const Offset(3.3, 4.4)]), + ); + expect( + c(FaceContourType.leftEye), + containsAllInOrder( + [const Offset(1.1, 2.2), const Offset(3.3, 4.4)]), + ); + expect( + c(FaceContourType.leftEyebrowBottom), + containsAllInOrder( + [const Offset(1.1, 2.2), const Offset(3.3, 4.4)]), + ); + expect( + c(FaceContourType.leftEyebrowTop), + containsAllInOrder( + [const Offset(1.1, 2.2), const Offset(3.3, 4.4)]), + ); + expect( + c(FaceContourType.lowerLipBottom), + containsAllInOrder( + [const Offset(1.1, 2.2), const Offset(3.3, 4.4)]), + ); + expect( + c(FaceContourType.lowerLipTop), + containsAllInOrder( + [const Offset(1.1, 2.2), const Offset(3.3, 4.4)]), + ); + expect( + c(FaceContourType.noseBottom), + containsAllInOrder( + [const Offset(1.1, 2.2), const Offset(3.3, 4.4)]), + ); + expect( + c(FaceContourType.noseBridge), + containsAllInOrder( + [const Offset(1.1, 2.2), const Offset(3.3, 4.4)]), + ); + expect( + c(FaceContourType.rightEye), + containsAllInOrder( + [const Offset(1.1, 2.2), const Offset(3.3, 4.4)]), + ); + expect( + c(FaceContourType.rightEyebrowBottom), + containsAllInOrder( + [const Offset(1.1, 2.2), const Offset(3.3, 4.4)]), + ); + expect( + c(FaceContourType.rightEyebrowTop), + containsAllInOrder( + [const Offset(1.1, 2.2), const Offset(3.3, 4.4)]), + ); + expect( + c(FaceContourType.upperLipBottom), + containsAllInOrder( + [const Offset(1.1, 2.2), const Offset(3.3, 4.4)]), + ); + expect( + c(FaceContourType.upperLipTop), + containsAllInOrder( + [const Offset(1.1, 2.2), const Offset(3.3, 4.4)]), + ); }); test('processImage with null landmark', () async {