From 761aebc784028d81e16917ec626dc3b0a57a7a7e Mon Sep 17 00:00:00 2001 From: LongCat is Looong <31859944+LongCatIsLooong@users.noreply.github.com> Date: Sat, 29 May 2021 00:29:58 -0700 Subject: [PATCH 1/4] [iOSTextInputPlugin] bypass UIKit floating cursor coordinates clamping --- .../Source/FlutterTextInputPlugin.mm | 67 +++++++++++++++++++ .../Source/FlutterTextInputPluginTest.m | 46 +++++++++++++ 2 files changed, 113 insertions(+) diff --git a/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm b/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm index 854a3ff7f4c1e..88ad772925bbd 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm @@ -22,6 +22,18 @@ // returns kInvalidFirstRect, iOS will not show the IME candidates view. const CGRect kInvalidFirstRect = {{-1, -1}, {9999, 9999}}; +// The `bounds` value a FlutterTextInputView returns when the floating cursor +// is activated in that view. +// +// DO NOT use extremely large values (such as CGFloat_MAX) in this rect, for that +// will significantly reduce the precision of the floating cursor's coordinates. +// +// It is recommended for this CGRect to be roughly centered at caretRectForPosition +// (which currently always return CGRectZero), so the initial floating cursor will +// be placed at (0, 0). +// See the comments in beginFloatingCursorAtPoint and caretRectForPosition. +const CGRect kSpacePanBounds = {{-2500, -2500}, {5000, 5000}}; + #pragma mark - TextInputConfiguration Field Names static NSString* const kSecureTextEntry = @"obscureText"; static NSString* const kKeyboardType = @"inputType"; @@ -505,6 +517,7 @@ @implementation FlutterTextInputView { const char* _selectionAffinity; FlutterTextRange* _selectedTextRange; CGRect _cachedFirstRect; + bool _isFloatingCursorActive; } @synthesize tokenizer = _tokenizer; @@ -524,6 +537,7 @@ - (instancetype)init { // Initialize with the zero matrix which is not // an affine transform. _editableTransform = CATransform3D(); + _isFloatingCursorActive = false; // UITextInputTraits _autocapitalizationType = UITextAutocapitalizationTypeSentences; @@ -539,6 +553,16 @@ - (instancetype)init { _smartQuotesType = UITextSmartQuotesTypeYes; _smartDashesType = UITextSmartDashesTypeYes; } + + // This makes sure UITextSelectionView.interactionAssistant is not nil so + // UITextSelectionView has access to this view (and its bounds). Otherwise + // floating cursor breaks: https://github.com/flutter/flutter/issues/70267. + if (@available(iOS 13.0, *)) { + UITextInteraction* interaction = + [UITextInteraction textInteractionForMode:UITextInteractionModeEditable]; + interaction.textInput = self; + [self addInteraction:interaction]; + } } return self; @@ -1071,9 +1095,23 @@ - (CGRect)firstRectForRange:(UITextRange*)range { - (CGRect)caretRectForPosition:(UITextPosition*)position { // TODO(cbracken) Implement. + + // As of iOS 14.4, this call is used by iOS's + // _UIKeyboardTextSelectionController to determine the position + // of the floating cursor when the user force touches the space + // bar to initiate floating cursor. + // + // It is recommended to return a value that's roughly the + // center of kSpacePanBounds to make sure the floating cursor + // has ample space in all directions and does not hit kSpacePanBounds. + // See the comments in beginFloatingCursorAtPoint. return CGRectZero; } +- (CGRect)bounds { + return _isFloatingCursorActive ? kSpacePanBounds : super.bounds; +} + - (UITextPosition*)closestPositionToPoint:(CGPoint)point { // TODO(cbracken) Implement. NSUInteger currentIndex = ((FlutterTextPosition*)_selectedTextRange.start).index; @@ -1097,18 +1135,47 @@ - (UITextRange*)characterRangeAtPoint:(CGPoint)point { } - (void)beginFloatingCursorAtPoint:(CGPoint)point { + // For "beginFloatingCursorAtPoint" and "updateFloatingCursorAtPoint", "point" is roughly: + // + // CGPoint( + // width >= 0 ? point.x.clamp(boundingBox.left, boundingBox.right) : point.x, + // height >= 0 ? point.y.clamp(boundingBox.top, boundingBox.bottom) : point.y, + // ) + // where + // point = keyboardPanGestureRecognizer.translationInView(textInputView) + + // caretRectForPosition boundingBox = self.convertRect(bounds, fromView:textInputView) bounds + // = self._selectionClipRect ?? self.bounds + // + // It's tricky to provide accurate "bounds" and "caretRectForPosition" so it's preferred to bypass + // the clamping and implement the same clamping logic in the framework where we have easy access + // to the bounding box of the input field and the caret location. + // + // The current implementation returns kSpacePanBounds for "bounds" when "_isFloatingCursorActive" + // is true. kSpacePanBounds centers "caretRectForPosition" so the floating cursor has enough + // clearance in all directions to move around. + // + // It seems impossible to use a negative "width" or "height", as the "convertRect" + // call always turns a CGRect's negative dimensions into non-negative values, e.g., + // (1, 2, -3, -4) would become (-2, -2, 3, 4). + NSAssert(!_isFloatingCursorActive, @"Another floating cursor is currently active."); + _isFloatingCursorActive = true; [_textInputDelegate updateFloatingCursor:FlutterFloatingCursorDragStateStart withClient:_textInputClient withPosition:@{@"X" : @(point.x), @"Y" : @(point.y)}]; } - (void)updateFloatingCursorAtPoint:(CGPoint)point { + NSAssert(_isFloatingCursorActive, + @"updateFloatingCursorAtPoint is called without an active floating cursor."); [_textInputDelegate updateFloatingCursor:FlutterFloatingCursorDragStateUpdate withClient:_textInputClient withPosition:@{@"X" : @(point.x), @"Y" : @(point.y)}]; } - (void)endFloatingCursor { + NSAssert(_isFloatingCursorActive, + @"endFloatingCursor is called without an active floating cursor."); + _isFloatingCursorActive = false; [_textInputDelegate updateFloatingCursor:FlutterFloatingCursorDragStateEnd withClient:_textInputClient withPosition:@{@"X" : @(0), @"Y" : @(0)}]; diff --git a/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.m b/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.m index 997daa2a0d48a..4919b03334557 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.m +++ b/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.m @@ -498,6 +498,52 @@ - (void)testUpdateFirstRectForRange { XCTAssertTrue(CGRectEqualToRect(kInvalidFirstRect, [inputView firstRectForRange:range])); } +#pragma mark - Floating Cursor - Tests + +- (void)testInputViewsHaveUIInteractions { + FlutterTextInputView* inputView = [[FlutterTextInputView alloc] init]; + NSLog(@"UIInteractions: %@", inputView.interactions); + XCTAssertGreaterThan(inputView.interactions.count, 0); +} + +- (void)testBoundsForFloatingCursor { + FlutterTextInputView* inputView = [[FlutterTextInputView alloc] init]; + + CGRect initialBounds = inputView.bounds; + // Make sure the initial bounds.size is not as large. + XCTAssertLessThan(inputView.bounds.size.width, 100); + XCTAssertLessThan(inputView.bounds.size.height, 100); + + [inputView beginFloatingCursorAtPoint:CGPointMake(123, 321)]; + CGRect bounds = inputView.bounds; + XCTAssertGreaterThan(bounds.size.width, 1000); + XCTAssertGreaterThan(bounds.size.height, 1000); + + // Verify the caret is centered. + XCTAssertEqual( + CGRectGetMidX(bounds), + CGRectGetMidX([inputView caretRectForPosition:[FlutterTextPosition positionWithIndex:1235]])); + XCTAssertEqual( + CGRectGetMidY(bounds), + CGRectGetMidY([inputView caretRectForPosition:[FlutterTextPosition positionWithIndex:4567]])); + + [inputView updateFloatingCursorAtPoint:CGPointMake(456, 654)]; + bounds = inputView.bounds; + XCTAssertGreaterThan(bounds.size.width, 1000); + XCTAssertGreaterThan(bounds.size.height, 1000); + + // Verify the caret is centered. + XCTAssertEqual( + CGRectGetMidX(bounds), + CGRectGetMidX([inputView caretRectForPosition:[FlutterTextPosition positionWithIndex:21]])); + XCTAssertEqual( + CGRectGetMidY(bounds), + CGRectGetMidY([inputView caretRectForPosition:[FlutterTextPosition positionWithIndex:42]])); + + [inputView endFloatingCursor]; + XCTAssertTrue(CGRectEqualToRect(initialBounds, inputView.bounds)); +} + #pragma mark - Autofill - Utilities - (NSMutableDictionary*)mutablePasswordTemplateCopy { From d19cb616ab7474af0dbc6977bf1afb13da8f9ca3 Mon Sep 17 00:00:00 2001 From: LongCat is Looong <31859944+LongCatIsLooong@users.noreply.github.com> Date: Sat, 29 May 2021 00:35:52 -0700 Subject: [PATCH 2/4] [iOSTextInputPlugin] bypass UIKit floating cursor coordinates clamping --- .../ios/framework/Source/FlutterTextInputPluginTest.m | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.m b/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.m index 4919b03334557..1bea44621a401 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.m +++ b/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.m @@ -501,9 +501,10 @@ - (void)testUpdateFirstRectForRange { #pragma mark - Floating Cursor - Tests - (void)testInputViewsHaveUIInteractions { - FlutterTextInputView* inputView = [[FlutterTextInputView alloc] init]; - NSLog(@"UIInteractions: %@", inputView.interactions); - XCTAssertGreaterThan(inputView.interactions.count, 0); + if (@available(iOS 13.0, *)) { + FlutterTextInputView* inputView = [[FlutterTextInputView alloc] init]; + XCTAssertGreaterThan(inputView.interactions.count, 0); + } } - (void)testBoundsForFloatingCursor { From b13caacf3782de1db6a2521e5a23324801e79cc9 Mon Sep 17 00:00:00 2001 From: LongCat is Looong <31859944+LongCatIsLooong@users.noreply.github.com> Date: Tue, 1 Jun 2021 11:18:04 -0700 Subject: [PATCH 3/4] add stopMocking back --- .../darwin/ios/framework/Source/FlutterTextInputPluginTest.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.m b/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.m index c271938fb02bf..867547740f9d5 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.m +++ b/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.m @@ -81,7 +81,7 @@ - (void)setUp { - (void)tearDown { [[[[textInputPlugin textInputView] superview] subviews] makeObjectsPerformSelector:@selector(removeFromSuperview)]; - + [engine stopMocking]; [super tearDown]; } From 659e74a759e3f34ba4bc9c147aa4573a6bf13299 Mon Sep 17 00:00:00 2001 From: LongCat is Looong <31859944+LongCatIsLooong@users.noreply.github.com> Date: Fri, 4 Jun 2021 01:32:26 -0700 Subject: [PATCH 4/4] fix typo --- .../ios/framework/Source/FlutterTextInputPlugin.mm | 12 ++++++------ .../framework/Source/FlutterTextInputPluginTest.m | 1 - 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm b/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm index 3d05dd1bb8e13..d009cec80690f 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm @@ -636,9 +636,9 @@ - (UITextContentType)textContentType { // from the view hierarchy) so that it may outlive the plugin/engine, // in which case _textInputDelegate will become a dangling pointer. -// The text input plugin needs to call decommision when it should +// The text input plugin needs to call decommission when it should // not have access to its FlutterTextInputDelegate any more. -- (void)decommision { +- (void)decommission { _decommissioned = YES; } @@ -1477,7 +1477,7 @@ - (void)hideTextInput { [self removeEnableFlutterTextInputViewAccessibilityTimer]; _activeView.accessibilityEnabled = NO; [_activeView resignFirstResponder]; - [_activeView decommision]; + [_activeView decommission]; [_activeView removeFromSuperview]; [_inputHider removeFromSuperview]; } @@ -1640,10 +1640,10 @@ - (UIView*)keyWindow { return _inputHider.subviews; } -// Decommisions (See the "decommision" method on FlutterTextInputView) and removes +// Decommissions (See the "decommission" method on FlutterTextInputView) and removes // every installed input field, unless it's in the current autofill context. // -// The active view will be decommisioned and removed from its superview too, if +// The active view will be decommissioned and removed from its superview too, if // includeActiveView is YES. // When clearText is YES, the text on the input fields will be set to empty before // they are removed from the view hierarchy, to avoid triggering autofill save. @@ -1663,7 +1663,7 @@ - (void)cleanUpViewHierarchy:(BOOL)includeActiveView if (clearText) { [inputView replaceRangeLocal:NSMakeRange(0, inputView.text.length) withText:@""]; } - [inputView decommision]; + [inputView decommission]; if (delayRemoval) { [inputView performSelector:@selector(removeFromSuperview) withObject:nil afterDelay:0.1]; } else { diff --git a/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.m b/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.m index 54fa854eb76b8..9cb0f836c42af 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.m +++ b/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.m @@ -19,7 +19,6 @@ - (void)setEditableTransform:(NSArray*)matrix; - (void)setTextInputState:(NSDictionary*)state; - (void)setMarkedRect:(CGRect)markedRect; - (void)updateEditingState; -- (void)decommisson; - (BOOL)isVisibleToAutofill; @end