From d50e7db44b464e89684852b959ea25bac27aeb96 Mon Sep 17 00:00:00 2001 From: LongCatIsLooong <31859944+LongCatIsLooong@users.noreply.github.com> Date: Fri, 9 Sep 2022 12:33:56 -0700 Subject: [PATCH] [macOS] Fix text input plugin editable transform (flutter/engine#35979) --- .../Source/FlutterTextInputPlugin.mm | 59 +++++-- .../Source/FlutterTextInputPluginTest.mm | 13 ++ .../Source/FlutterTextInputPlugin.mm | 63 ++++++-- .../Source/FlutterTextInputPluginTest.mm | 152 +++++++++++++++++- 4 files changed, 260 insertions(+), 27 deletions(-) diff --git a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm index 8a3e39d7a8c..edb7d70a33d 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm +++ b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm @@ -1511,11 +1511,49 @@ static BOOL IsSelectionRectCloserToPoint(CGPoint point, _cachedFirstRect = kInvalidFirstRect; } +// Returns the bounding CGRect of the transformed incomingRect, in the view's +// coordinates. +- (CGRect)localRectFromFrameworkTransform:(CGRect)incomingRect { + CGPoint points[] = { + incomingRect.origin, + CGPointMake(incomingRect.origin.x, incomingRect.origin.y + incomingRect.size.height), + CGPointMake(incomingRect.origin.x + incomingRect.size.width, incomingRect.origin.y), + CGPointMake(incomingRect.origin.x + incomingRect.size.width, + incomingRect.origin.y + incomingRect.size.height)}; + + CGPoint origin = CGPointMake(CGFLOAT_MAX, CGFLOAT_MAX); + CGPoint farthest = CGPointMake(-CGFLOAT_MAX, -CGFLOAT_MAX); + + for (int i = 0; i < 4; i++) { + const CGPoint point = points[i]; + + CGFloat x = _editableTransform.m11 * point.x + _editableTransform.m21 * point.y + + _editableTransform.m41; + CGFloat y = _editableTransform.m12 * point.x + _editableTransform.m22 * point.y + + _editableTransform.m42; + + const CGFloat w = _editableTransform.m14 * point.x + _editableTransform.m24 * point.y + + _editableTransform.m44; + + if (w == 0.0) { + return kInvalidFirstRect; + } else if (w != 1.0) { + x /= w; + y /= w; + } + + origin.x = MIN(origin.x, x); + origin.y = MIN(origin.y, y); + farthest.x = MAX(farthest.x, x); + farthest.y = MAX(farthest.y, y); + } + return CGRectMake(origin.x, origin.y, farthest.x - origin.x, farthest.y - origin.y); +} + // The following methods are required to support force-touch cursor positioning // and to position the // candidates view for multi-stage input methods (e.g., Japanese) when using a // physical keyboard. - - (CGRect)firstRectForRange:(UITextRange*)range { NSAssert([range.start isKindOfClass:[FlutterTextPosition class]], @"Expected a FlutterTextPosition for range.start (got %@).", [range.start class]); @@ -1524,22 +1562,21 @@ static BOOL IsSelectionRectCloserToPoint(CGPoint point, NSUInteger start = ((FlutterTextPosition*)range.start).index; NSUInteger end = ((FlutterTextPosition*)range.end).index; if (_markedTextRange != nil) { - // The candidates view can't be shown if _editableTransform is not affine, - // or markedRect is invalid. - if (CGRectEqualToRect(kInvalidFirstRect, _markedRect) || - !CATransform3DIsAffine(_editableTransform)) { + // The candidates view can't be shown if the framework has not sent the + // first caret rect. + if (CGRectEqualToRect(kInvalidFirstRect, _markedRect)) { return kInvalidFirstRect; } if (CGRectEqualToRect(_cachedFirstRect, kInvalidFirstRect)) { // If the width returned is too small, that means the framework sent us - // the caret rect instead of the marked text rect. Expand it to 0.1 so - // the IME candidates view show up. - double nonZeroWidth = MAX(_markedRect.size.width, 0.1); + // the caret rect instead of the marked text rect. Expand it to 0.2 so + // the IME candidates view would show up. CGRect rect = _markedRect; - rect.size = CGSizeMake(nonZeroWidth, rect.size.height); - _cachedFirstRect = - CGRectApplyAffineTransform(rect, CATransform3DGetAffineTransform(_editableTransform)); + if (CGRectIsEmpty(rect)) { + rect = CGRectInset(rect, -0.1, 0); + } + _cachedFirstRect = [self localRectFromFrameworkTransform:rect]; } return _cachedFirstRect; diff --git a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.mm b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.mm index aec571a8125..09c307ef0db 100644 --- a/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.mm +++ b/engine/src/flutter/shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.mm @@ -1167,6 +1167,13 @@ FLUTTER_ASSERT_ARC // yOffset = 200. NSArray* yOffsetMatrix = @[ @1, @0, @0, @0, @0, @1, @0, @0, @0, @0, @1, @0, @0, @200, @0, @1 ]; NSArray* zeroMatrix = @[ @0, @0, @0, @0, @0, @0, @0, @0, @0, @0, @0, @0, @0, @0, @0, @0 ]; + // This matrix can be generated by running this dart code snippet: + // Matrix4.identity()..scale(3.0)..rotateZ(math.pi/2)..translate(1.0, 2.0, + // 3.0); + NSArray* affineMatrix = @[ + @(0.0), @(3.0), @(0.0), @(0.0), @(-3.0), @(0.0), @(0.0), @(0.0), @(0.0), @(0.0), @(3.0), @(0.0), + @(-6.0), @(3.0), @(9.0), @(1.0) + ]; // Invalid since we don't have the transform or the rect. XCTAssertTrue(CGRectEqualToRect(kInvalidFirstRect, [inputView firstRectForRange:range])); @@ -1200,6 +1207,12 @@ FLUTTER_ASSERT_ARC // Invalid marked rect is invalid. XCTAssertTrue(CGRectEqualToRect(kInvalidFirstRect, [inputView firstRectForRange:range])); XCTAssertTrue(CGRectEqualToRect(kInvalidFirstRect, [inputView firstRectForRange:range])); + + // Use a 3d affine transform that does 3d-scaling, z-index rotating and 3d translation. + [inputView setEditableTransform:affineMatrix]; + [inputView setMarkedRect:testRect]; + XCTAssertTrue( + CGRectEqualToRect(CGRectMake(-306, 3, 300, 300), [inputView firstRectForRange:range])); } - (void)testFirstRectForRangeReturnsCorrectSelectionRect { diff --git a/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputPlugin.mm b/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputPlugin.mm index 698d6fbbd03..d6687772bc7 100644 --- a/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputPlugin.mm +++ b/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputPlugin.mm @@ -824,24 +824,57 @@ static char markerKey; return @[]; } -- (NSRect)firstRectForCharacterRange:(NSRange)range actualRange:(NSRangePointer)actualRange { - if (!self.flutterViewController.viewLoaded) { - return CGRectZero; +// Returns the bounding CGRect of the transformed incomingRect, in screen +// coordinates. +- (CGRect)screenRectFromFrameworkTransform:(CGRect)incomingRect { + CGPoint points[] = { + incomingRect.origin, + CGPointMake(incomingRect.origin.x, incomingRect.origin.y + incomingRect.size.height), + CGPointMake(incomingRect.origin.x + incomingRect.size.width, incomingRect.origin.y), + CGPointMake(incomingRect.origin.x + incomingRect.size.width, + incomingRect.origin.y + incomingRect.size.height)}; + + CGPoint origin = CGPointMake(CGFLOAT_MAX, CGFLOAT_MAX); + CGPoint farthest = CGPointMake(-CGFLOAT_MAX, -CGFLOAT_MAX); + + for (int i = 0; i < 4; i++) { + const CGPoint point = points[i]; + + CGFloat x = _editableTransform.m11 * point.x + _editableTransform.m21 * point.y + + _editableTransform.m41; + CGFloat y = _editableTransform.m12 * point.x + _editableTransform.m22 * point.y + + _editableTransform.m42; + + const CGFloat w = _editableTransform.m14 * point.x + _editableTransform.m24 * point.y + + _editableTransform.m44; + + if (w == 0.0) { + return CGRectZero; + } else if (w != 1.0) { + x /= w; + y /= w; + } + + origin.x = MIN(origin.x, x); + origin.y = MIN(origin.y, y); + farthest.x = MAX(farthest.x, x); + farthest.y = MAX(farthest.y, y); } + + const NSView* fromView = self.flutterViewController.flutterView; + const CGRect rectInWindow = [fromView + convertRect:CGRectMake(origin.x, origin.y, farthest.x - origin.x, farthest.y - origin.y) + toView:nil]; + NSWindow* window = fromView.window; + return window ? [window convertRectToScreen:rectInWindow] : rectInWindow; +} + +- (NSRect)firstRectForCharacterRange:(NSRange)range actualRange:(NSRangePointer)actualRange { // This only determines position of caret instead of any arbitrary range, but it's enough // to properly position accent selection popup - if (CATransform3DIsAffine(_editableTransform) && !CGRectEqualToRect(_caretRect, CGRectNull)) { - CGRect rect = - CGRectApplyAffineTransform(_caretRect, CATransform3DGetAffineTransform(_editableTransform)); - - // convert to window coordinates - rect = [self.flutterViewController.flutterView convertRect:rect toView:nil]; - - // convert to screen coordinates - return [self.flutterViewController.flutterView.window convertRectToScreen:rect]; - } else { - return CGRectZero; - } + return !self.flutterViewController.viewLoaded || CGRectEqualToRect(_caretRect, CGRectNull) + ? CGRectZero + : [self screenRectFromFrameworkTransform:_caretRect]; } - (NSUInteger)characterIndexForPoint:(NSPoint)point { diff --git a/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputPluginTest.mm b/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputPluginTest.mm index 83f50a34032..4efb4d3e30b 100644 --- a/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputPluginTest.mm +++ b/engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterTextInputPluginTest.mm @@ -438,7 +438,6 @@ }]; NSRect rect = [plugin firstRectForCharacterRange:NSMakeRange(0, 0) actualRange:nullptr]; - @try { OCMVerify( // NOLINT(google-objc-avoid-throwing-exception) [windowMock convertRectToScreen:NSMakeRect(28, 10, 2, 19)]); @@ -449,6 +448,148 @@ return NSEqualRects(rect, NSMakeRect(38, 20, 2, 19)); } +- (bool)testFirstRectForCharacterRangeAtInfinity { + id engineMock = OCMClassMock([FlutterEngine class]); + id binaryMessengerMock = OCMProtocolMock(@protocol(FlutterBinaryMessenger)); + OCMStub( // NOLINT(google-objc-avoid-throwing-exception) + [engineMock binaryMessenger]) + .andReturn(binaryMessengerMock); + FlutterViewController* controllerMock = OCMClassMock([FlutterViewController class]); + OCMStub( // NOLINT(google-objc-avoid-throwing-exception) + [controllerMock engine]) + .andReturn(engineMock); + + id viewMock = OCMClassMock([NSView class]); + OCMStub( // NOLINT(google-objc-avoid-throwing-exception) + [viewMock bounds]) + .andReturn(NSMakeRect(0, 0, 200, 200)); + OCMStub( // NOLINT(google-objc-avoid-throwing-exception) + controllerMock.viewLoaded) + .andReturn(YES); + OCMStub( // NOLINT(google-objc-avoid-throwing-exception) + [controllerMock flutterView]) + .andReturn(viewMock); + + id windowMock = OCMClassMock([NSWindow class]); + OCMStub( // NOLINT(google-objc-avoid-throwing-exception) + [viewMock window]) + .andReturn(windowMock); + + FlutterTextInputPlugin* plugin = + [[FlutterTextInputPlugin alloc] initWithViewController:controllerMock]; + + FlutterMethodCall* call = [FlutterMethodCall + methodCallWithMethodName:@"TextInput.setEditableSizeAndTransform" + arguments:@{ + @"height" : @(20.0), + // Projects all points to infinity. + @"transform" : @[ + @(1.0), @(0.0), @(0.0), @(0.0), @(0.0), @(1.0), @(0.0), @(0.0), @(0.0), + @(0.0), @(1.0), @(0.0), @(20.0), @(10.0), @(0.0), @(0.0) + ], + @"width" : @(400.0), + }]; + + [plugin handleMethodCall:call + result:^(id){ + }]; + + call = [FlutterMethodCall methodCallWithMethodName:@"TextInput.setCaretRect" + arguments:@{ + @"height" : @(19.0), + @"width" : @(2.0), + @"x" : @(8.0), + @"y" : @(0.0), + }]; + + [plugin handleMethodCall:call + result:^(id){ + }]; + + NSRect rect = [plugin firstRectForCharacterRange:NSMakeRange(0, 0) actualRange:nullptr]; + return NSEqualRects(rect, CGRectZero); +} + +- (bool)testFirstRectForCharacterRangeWithEsotericAffineTransform { + id engineMock = OCMClassMock([FlutterEngine class]); + id binaryMessengerMock = OCMProtocolMock(@protocol(FlutterBinaryMessenger)); + OCMStub( // NOLINT(google-objc-avoid-throwing-exception) + [engineMock binaryMessenger]) + .andReturn(binaryMessengerMock); + FlutterViewController* controllerMock = OCMClassMock([FlutterViewController class]); + OCMStub( // NOLINT(google-objc-avoid-throwing-exception) + [controllerMock engine]) + .andReturn(engineMock); + + id viewMock = OCMClassMock([NSView class]); + OCMStub( // NOLINT(google-objc-avoid-throwing-exception) + [viewMock bounds]) + .andReturn(NSMakeRect(0, 0, 200, 200)); + OCMStub( // NOLINT(google-objc-avoid-throwing-exception) + controllerMock.viewLoaded) + .andReturn(YES); + OCMStub( // NOLINT(google-objc-avoid-throwing-exception) + [controllerMock flutterView]) + .andReturn(viewMock); + + id windowMock = OCMClassMock([NSWindow class]); + OCMStub( // NOLINT(google-objc-avoid-throwing-exception) + [viewMock window]) + .andReturn(windowMock); + + OCMExpect( // NOLINT(google-objc-avoid-throwing-exception) + [viewMock convertRect:NSMakeRect(-18, 6, 3, 3) toView:nil]) + .andReturn(NSMakeRect(-18, 6, 3, 3)); + + OCMExpect( // NOLINT(google-objc-avoid-throwing-exception) + [windowMock convertRectToScreen:NSMakeRect(-18, 6, 3, 3)]) + .andReturn(NSMakeRect(-18, 6, 3, 3)); + + FlutterTextInputPlugin* plugin = + [[FlutterTextInputPlugin alloc] initWithViewController:controllerMock]; + + FlutterMethodCall* call = [FlutterMethodCall + methodCallWithMethodName:@"TextInput.setEditableSizeAndTransform" + arguments:@{ + @"height" : @(20.0), + // This matrix can be generated by running this dart code snippet: + // Matrix4.identity()..scale(3.0)..rotateZ(math.pi/2)..translate(1.0, 2.0, + // 3.0); + @"transform" : @[ + @(0.0), @(3.0), @(0.0), @(0.0), @(-3.0), @(0.0), @(0.0), @(0.0), @(0.0), + @(0.0), @(3.0), @(0.0), @(-6.0), @(3.0), @(9.0), @(1.0) + ], + @"width" : @(400.0), + }]; + + [plugin handleMethodCall:call + result:^(id){ + }]; + + call = [FlutterMethodCall methodCallWithMethodName:@"TextInput.setCaretRect" + arguments:@{ + @"height" : @(1.0), + @"width" : @(1.0), + @"x" : @(1.0), + @"y" : @(3.0), + }]; + + [plugin handleMethodCall:call + result:^(id){ + }]; + + NSRect rect = [plugin firstRectForCharacterRange:NSMakeRange(0, 0) actualRange:nullptr]; + + @try { + OCMVerify( // NOLINT(google-objc-avoid-throwing-exception) + [windowMock convertRectToScreen:NSMakeRect(-18, 6, 3, 3)]); + } @catch (...) { + return false; + } + + return NSEqualRects(rect, NSMakeRect(-18, 6, 3, 3)); +} + - (bool)testSetEditingStateWithTextEditingDelta { id engineMock = OCMClassMock([FlutterEngine class]); id binaryMessengerMock = OCMProtocolMock(@protocol(FlutterBinaryMessenger)); @@ -1226,6 +1367,15 @@ TEST(FlutterTextInputPluginTest, TestFirstRectForCharacterRange) { ASSERT_TRUE([[FlutterInputPluginTestObjc alloc] testFirstRectForCharacterRange]); } +TEST(FlutterTextInputPluginTest, TestFirstRectForCharacterRangeAtInfinity) { + ASSERT_TRUE([[FlutterInputPluginTestObjc alloc] testFirstRectForCharacterRangeAtInfinity]); +} + +TEST(FlutterTextInputPluginTest, TestFirstRectForCharacterRangeWithEsotericAffineTransform) { + ASSERT_TRUE([[FlutterInputPluginTestObjc alloc] + testFirstRectForCharacterRangeWithEsotericAffineTransform]); +} + TEST(FlutterTextInputPluginTest, TestSetEditingStateWithTextEditingDelta) { ASSERT_TRUE([[FlutterInputPluginTestObjc alloc] testSetEditingStateWithTextEditingDelta]); }