diff --git a/packages/firebase_ml_vision/CHANGELOG.md b/packages/firebase_ml_vision/CHANGELOG.md index 058b0d053560..f25d393d393f 100644 --- a/packages/firebase_ml_vision/CHANGELOG.md +++ b/packages/firebase_ml_vision/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.9.1 + +* Add support for cloud text recognizer. + ## 0.9.0+3 * Automatically use version from pubspec.yaml when reporting usage to Firebase. diff --git a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextRecognizer.java b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextRecognizer.java index 55aa3c8bc5d7..9a0f82343ff4 100644 --- a/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextRecognizer.java +++ b/packages/firebase_ml_vision/android/src/main/java/io/flutter/plugins/firebasemlvision/TextRecognizer.java @@ -21,7 +21,15 @@ public class TextRecognizer implements Detector { private final FirebaseVisionTextRecognizer recognizer; TextRecognizer(FirebaseVision vision, Map options) { - recognizer = vision.getOnDeviceTextRecognizer(); + final String modelType = (String) options.get("modelType"); + if (modelType.equals("onDevice")) { + recognizer = vision.getOnDeviceTextRecognizer(); + } else if (modelType.equals("cloud")) { + recognizer = vision.getCloudTextRecognizer(); + } else { + final String message = String.format("No model for type: %s", modelType); + throw new IllegalArgumentException(message); + } } @Override diff --git a/packages/firebase_ml_vision/example/lib/camera_preview_scanner.dart b/packages/firebase_ml_vision/example/lib/camera_preview_scanner.dart index c8c5eedf7a64..f40e20d6591c 100644 --- a/packages/firebase_ml_vision/example/lib/camera_preview_scanner.dart +++ b/packages/firebase_ml_vision/example/lib/camera_preview_scanner.dart @@ -29,6 +29,8 @@ class _CameraPreviewScannerState extends State { final ImageLabeler _cloudImageLabeler = FirebaseVision.instance.cloudImageLabeler(); final TextRecognizer _recognizer = FirebaseVision.instance.textRecognizer(); + final TextRecognizer _cloudRecognizer = + FirebaseVision.instance.cloudTextRecognizer(); @override void initState() { @@ -72,6 +74,8 @@ class _CameraPreviewScannerState extends State { switch (_currentDetector) { case Detector.text: return _recognizer.processImage; + case Detector.cloudText: + return _cloudRecognizer.processImage; case Detector.barcode: return _barcodeDetector.detectInImage; case Detector.label: @@ -119,7 +123,8 @@ class _CameraPreviewScannerState extends State { painter = LabelDetectorPainter(imageSize, _scanResults); break; default: - assert(_currentDetector == Detector.text); + assert(_currentDetector == Detector.text || + _currentDetector == Detector.cloudText); if (_scanResults is! VisionText) return noResultsText; painter = TextDetectorPainter(imageSize, _scanResults); } @@ -200,6 +205,10 @@ class _CameraPreviewScannerState extends State { child: Text('Detect Text'), value: Detector.text, ), + const PopupMenuItem( + child: Text('Detect Cloud Text'), + value: Detector.cloudText, + ), ], ), ], @@ -222,6 +231,7 @@ class _CameraPreviewScannerState extends State { _imageLabeler.close(); _cloudImageLabeler.close(); _recognizer.close(); + _cloudRecognizer.close(); }); _currentDetector = null; diff --git a/packages/firebase_ml_vision/example/lib/detector_painters.dart b/packages/firebase_ml_vision/example/lib/detector_painters.dart index 684503b1086c..d8d7316f2509 100644 --- a/packages/firebase_ml_vision/example/lib/detector_painters.dart +++ b/packages/firebase_ml_vision/example/lib/detector_painters.dart @@ -7,7 +7,7 @@ import 'dart:ui' as ui; import 'package:firebase_ml_vision/firebase_ml_vision.dart'; import 'package:flutter/material.dart'; -enum Detector { barcode, face, label, cloudLabel, text } +enum Detector { barcode, face, label, cloudLabel, text, cloudText } class BarcodeDetectorPainter extends CustomPainter { BarcodeDetectorPainter(this.absoluteImageSize, this.barcodeLocations); diff --git a/packages/firebase_ml_vision/example/lib/picture_scanner.dart b/packages/firebase_ml_vision/example/lib/picture_scanner.dart index 88b1c5a1f0bb..182a55a46020 100644 --- a/packages/firebase_ml_vision/example/lib/picture_scanner.dart +++ b/packages/firebase_ml_vision/example/lib/picture_scanner.dart @@ -29,6 +29,8 @@ class _PictureScannerState extends State { final ImageLabeler _cloudImageLabeler = FirebaseVision.instance.cloudImageLabeler(); final TextRecognizer _recognizer = FirebaseVision.instance.textRecognizer(); + final TextRecognizer _cloudRecognizer = + FirebaseVision.instance.cloudTextRecognizer(); Future _getAndScanImage() async { setState(() { @@ -93,6 +95,9 @@ class _PictureScannerState extends State { case Detector.text: results = await _recognizer.processImage(visionImage); break; + case Detector.cloudText: + results = await _cloudRecognizer.processImage(visionImage); + break; default: return; } @@ -121,6 +126,9 @@ class _PictureScannerState extends State { case Detector.text: painter = TextDetectorPainter(_imageSize, results); break; + case Detector.cloudText: + painter = TextDetectorPainter(_imageSize, results); + break; default: break; } @@ -185,6 +193,10 @@ class _PictureScannerState extends State { child: Text('Detect Text'), value: Detector.text, ), + const PopupMenuItem( + child: Text('Detect Cloud Text'), + value: Detector.cloudText, + ), ], ), ], @@ -207,6 +219,7 @@ class _PictureScannerState extends State { _imageLabeler.close(); _cloudImageLabeler.close(); _recognizer.close(); + _cloudRecognizer.close(); super.dispose(); } } diff --git a/packages/firebase_ml_vision/ios/Classes/TextRecognizer.m b/packages/firebase_ml_vision/ios/Classes/TextRecognizer.m index fa9bdd3158e2..cc8cc9306a12 100644 --- a/packages/firebase_ml_vision/ios/Classes/TextRecognizer.m +++ b/packages/firebase_ml_vision/ios/Classes/TextRecognizer.m @@ -8,7 +8,17 @@ @implementation TextRecognizer - (instancetype)initWithVision:(FIRVision *)vision options:(NSDictionary *)options { self = [super init]; if (self) { - _recognizer = [vision onDeviceTextRecognizer]; + if ([@"onDevice" isEqualToString:options[@"modelType"]]) { + _recognizer = [vision onDeviceTextRecognizer]; + } else if ([@"cloud" isEqualToString:options[@"modelType"]]) { + _recognizer = [vision cloudTextRecognizer]; + } else { + NSString *reason = + [NSString stringWithFormat:@"Invalid model type: %@", options[@"modelType"]]; + @throw [[NSException alloc] initWithName:NSInvalidArgumentException + reason:reason + userInfo:nil]; + } } return self; } diff --git a/packages/firebase_ml_vision/lib/src/firebase_vision.dart b/packages/firebase_ml_vision/lib/src/firebase_vision.dart index 6f6b0446e3ea..d0f593cff8f8 100644 --- a/packages/firebase_ml_vision/lib/src/firebase_vision.dart +++ b/packages/firebase_ml_vision/lib/src/firebase_vision.dart @@ -67,7 +67,12 @@ class FirebaseVision { } /// Creates an instance of [TextRecognizer]. - TextRecognizer textRecognizer() => TextRecognizer._(nextHandle++); + TextRecognizer textRecognizer() { + return TextRecognizer._( + modelType: ModelType.onDevice, + handle: nextHandle++, + ); + } /// Creates a cloud instance of [ImageLabeler]. ImageLabeler cloudImageLabeler([CloudImageLabelerOptions options]) { @@ -77,6 +82,14 @@ class FirebaseVision { handle: nextHandle++, ); } + + /// Creates a cloud instance of [TextRecognizer]. + TextRecognizer cloudTextRecognizer() { + return TextRecognizer._( + modelType: ModelType.cloud, + handle: nextHandle++, + ); + } } /// Represents an image object used for both on-device and cloud API detectors. diff --git a/packages/firebase_ml_vision/lib/src/text_recognizer.dart b/packages/firebase_ml_vision/lib/src/text_recognizer.dart index 888f5fd5e03b..98936f66a906 100644 --- a/packages/firebase_ml_vision/lib/src/text_recognizer.dart +++ b/packages/firebase_ml_vision/lib/src/text_recognizer.dart @@ -19,7 +19,13 @@ part of firebase_ml_vision; /// await textRecognizer.processImage(image); /// ``` class TextRecognizer { - TextRecognizer._(this._handle); + TextRecognizer._({ + @required this.modelType, + @required int handle, + }) : _handle = handle, + assert(modelType != null); + + final ModelType modelType; final int _handle; bool _hasBeenOpened = false; @@ -35,7 +41,9 @@ class TextRecognizer { 'TextRecognizer#processImage', { 'handle': _handle, - 'options': {}, + 'options': { + 'modelType': _enumToString(modelType), + }, }..addAll(visionImage._serialize()), ); diff --git a/packages/firebase_ml_vision/pubspec.yaml b/packages/firebase_ml_vision/pubspec.yaml index 765b72b16a09..c2138b1866d0 100644 --- a/packages/firebase_ml_vision/pubspec.yaml +++ b/packages/firebase_ml_vision/pubspec.yaml @@ -2,7 +2,7 @@ name: firebase_ml_vision description: Flutter plugin for Firebase machine learning vision services. author: Flutter Team homepage: https://github.com/flutter/plugins/tree/master/packages/firebase_ml_vision -version: 0.9.0+3 +version: 0.9.1 dependencies: flutter: diff --git a/packages/firebase_ml_vision/test/firebase_ml_vision_test.dart b/packages/firebase_ml_vision/test/firebase_ml_vision_test.dart index 9420ddad85a0..85e069774b9a 100644 --- a/packages/firebase_ml_vision/test/firebase_ml_vision_test.dart +++ b/packages/firebase_ml_vision/test/firebase_ml_vision_test.dart @@ -83,7 +83,9 @@ void main() { }, ], }, - 'options': {}, + 'options': { + 'modelType': 'onDevice', + }, }, ), ]); @@ -778,7 +780,268 @@ void main() { expect(block.confidence, 0.5); block = text.blocks[1]; + // TODO(jackson): Use const Rect when available in minimum Flutter SDK + // ignore: prefer_const_constructors + expect(block.boundingBox, Rect.fromLTWH(14.0, 13.0, 16.0, 15.0)); + expect(block.text, 'hello'); + expect(block.cornerPoints, const [ + Offset(18.0, 17.0), + Offset(20.0, 19.0), + ]); + expect(block.confidence, 0.6); + }); + }); + + group('$TextLine', () { + test('processImage', () async { + final VisionText text = await recognizer.processImage(image); + + TextLine line = text.blocks[0].lines[0]; + // TODO(jackson): Use const Rect when available in minimum Flutter SDK + // ignore: prefer_const_constructors + expect(line.boundingBox, Rect.fromLTWH(5, 6, 7, 8)); + expect(line.text, 'friend'); + expect(line.cornerPoints, const [ + Offset(9.0, 10.0), + Offset(11.0, 12.0), + ]); + expect(line.recognizedLanguages, hasLength(2)); + expect(line.recognizedLanguages[0].languageCode, 'ef'); + expect(line.recognizedLanguages[1].languageCode, 'gh'); + expect(line.confidence, 0.3); + + line = text.blocks[0].lines[1]; + // TODO(jackson): Use const Rect when available in minimum Flutter SDK + // ignore: prefer_const_constructors + expect(line.boundingBox, Rect.fromLTWH(8.0, 7.0, 4.0, 5.0)); + expect(line.text, 'how'); + expect(line.cornerPoints, const [ + Offset(10.0, 9.0), + Offset(12.0, 11.0), + ]); + expect(line.confidence, 0.4); + }); + }); + + group('$TextElement', () { + test('processImage', () async { + final VisionText text = await recognizer.processImage(image); + + TextElement element = text.blocks[0].lines[0].elements[0]; // ignore: prefer_const_constructors + expect(element.boundingBox, Rect.fromLTWH(1.0, 2.0, 3.0, 4.0)); + expect(element.text, 'hello'); + expect(element.cornerPoints, const [ + Offset(5.0, 6.0), + Offset(7.0, 8.0), + ]); + expect(element.recognizedLanguages, hasLength(2)); + expect(element.recognizedLanguages[0].languageCode, 'ab'); + expect(element.recognizedLanguages[1].languageCode, 'cd'); + expect(element.confidence, 0.1); + + element = text.blocks[0].lines[0].elements[1]; + // TODO(jackson): Use const Rect when available in minimum Flutter SDK + // ignore: prefer_const_constructors + expect(element.boundingBox, Rect.fromLTWH(4.0, 3.0, 2.0, 1.0)); + expect(element.text, 'my'); + expect(element.cornerPoints, const [ + Offset(6.0, 5.0), + Offset(8.0, 7.0), + ]); + expect(element.confidence, 0.2); + }); + }); + + test('processImage', () async { + final VisionText text = await recognizer.processImage(image); + + expect(text.text, 'testext'); + expect(log, [ + isMethodCall( + 'TextRecognizer#processImage', + arguments: { + 'handle': 0, + 'type': 'file', + 'path': 'empty', + 'bytes': null, + 'metadata': null, + 'options': { + 'modelType': 'onDevice', + }, + }, + ), + ]); + }); + + test('processImage no bounding box', () async { + returnValue = { + 'blocks': [ + { + 'text': '', + 'points': [], + 'recognizedLanguages': [], + 'lines': [], + }, + ], + }; + + final VisionText text = await recognizer.processImage(image); + + final TextBlock block = text.blocks[0]; + expect(block.boundingBox, null); + }); + }); + + group('Cloud $TextRecognizer', () { + TextRecognizer recognizer; + final FirebaseVisionImage image = FirebaseVisionImage.fromFilePath( + 'empty', + ); + + setUp(() { + recognizer = FirebaseVision.instance.cloudTextRecognizer(); + final List elements = [ + { + 'text': 'hello', + 'left': 1.0, + 'top': 2.0, + 'width': 3.0, + 'height': 4.0, + 'points': [ + [5.0, 6.0], + [7.0, 8.0], + ], + 'recognizedLanguages': [ + { + 'languageCode': 'ab', + }, + { + 'languageCode': 'cd', + } + ], + 'confidence': 0.1, + }, + { + 'text': 'my', + 'left': 4.0, + 'top': 3.0, + 'width': 2.0, + 'height': 1.0, + 'points': [ + [6.0, 5.0], + [8.0, 7.0], + ], + 'recognizedLanguages': [], + 'confidence': 0.2, + }, + ]; + + final List lines = [ + { + 'text': 'friend', + 'left': 5.0, + 'top': 6.0, + 'width': 7.0, + 'height': 8.0, + 'points': [ + [9.0, 10.0], + [11.0, 12.0], + ], + 'recognizedLanguages': [ + { + 'languageCode': 'ef', + }, + { + 'languageCode': 'gh', + } + ], + 'elements': elements, + 'confidence': 0.3, + }, + { + 'text': 'how', + 'left': 8.0, + 'top': 7.0, + 'width': 4.0, + 'height': 5.0, + 'points': [ + [10.0, 9.0], + [12.0, 11.0], + ], + 'recognizedLanguages': [], + 'elements': [], + 'confidence': 0.4, + }, + ]; + + final List blocks = [ + { + 'text': 'friend', + 'left': 13.0, + 'top': 14.0, + 'width': 15.0, + 'height': 16.0, + 'points': [ + [17.0, 18.0], + [19.0, 20.0], + ], + 'recognizedLanguages': [ + { + 'languageCode': 'ij', + }, + { + 'languageCode': 'kl', + } + ], + 'lines': lines, + 'confidence': 0.5, + }, + { + 'text': 'hello', + 'left': 14.0, + 'top': 13.0, + 'width': 16.0, + 'height': 15.0, + 'points': [ + [18.0, 17.0], + [20.0, 19.0], + ], + 'recognizedLanguages': [], + 'lines': [], + 'confidence': 0.6, + }, + ]; + + final dynamic visionText = { + 'text': 'testext', + 'blocks': blocks, + }; + + returnValue = visionText; + }); + + group('$TextBlock', () { + test('processImage', () async { + final VisionText text = await recognizer.processImage(image); + + expect(text.blocks, hasLength(2)); + + TextBlock block = text.blocks[0]; + // TODO(jackson): Use const Rect when available in minimum Flutter SDK + // ignore: prefer_const_constructors + expect(block.boundingBox, Rect.fromLTWH(13.0, 14.0, 15.0, 16.0)); + expect(block.text, 'friend'); + expect(block.cornerPoints, const [ + Offset(17.0, 18.0), + Offset(19.0, 20.0), + ]); + expect(block.recognizedLanguages, hasLength(2)); + expect(block.recognizedLanguages[0].languageCode, 'ij'); + expect(block.recognizedLanguages[1].languageCode, 'kl'); + expect(block.confidence, 0.5); + + block = text.blocks[1]; // TODO(jackson): Use const Rect when available in minimum Flutter SDK // ignore: prefer_const_constructors expect(block.boundingBox, Rect.fromLTWH(14.0, 13.0, 16.0, 15.0)); @@ -827,6 +1090,7 @@ void main() { final VisionText text = await recognizer.processImage(image); TextElement element = text.blocks[0].lines[0].elements[0]; + // TODO(jackson): Use const Rect when available in minimum Flutter SDK // ignore: prefer_const_constructors expect(element.boundingBox, Rect.fromLTWH(1.0, 2.0, 3.0, 4.0)); expect(element.text, 'hello'); @@ -865,7 +1129,9 @@ void main() { 'path': 'empty', 'bytes': null, 'metadata': null, - 'options': {}, + 'options': { + 'modelType': 'cloud', + }, }, ), ]);