[macOS] Force clipping to path when platform view clip rect is rotated (flutter/engine#42539)

Normally when platform view is clipped to a simple rect we rely on
clipping to container layer bounds. However when the clip rect is
rotated the container layer is expanded accordingly and clipping to path
must be used instead.

Fixes https://github.com/flutter/flutter/issues/128175

engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterMutatorView.mm

@@ -92,6 +92,14 @@ CATransform3D ToCATransform3D(const FlutterTransformation& t) {
   return transform;
 }
 
+bool AffineTransformIsOnlyScaleOrTranslate(const CGAffineTransform& transform) {
+  return transform.b == 0 && transform.c == 0;
+}
+
+bool IsZeroSize(const FlutterSize size) {
+  return size.width == 0 && size.height == 0;
+}
+
 CGRect FromFlutterRect(const FlutterRect& rect) {
   return CGRectMake(rect.left, rect.top, rect.right - rect.left, rect.bottom - rect.top);
 }
@@ -182,6 +190,13 @@ bool RoundRectCornerIntersects(const FlutterRoundedRect& roundRect, const Flutte
 }
 
 CGPathRef PathFromRoundedRect(const FlutterRoundedRect& roundedRect) {
+  if (IsZeroSize(roundedRect.lower_left_corner_radius) &&
+      IsZeroSize(roundedRect.lower_right_corner_radius) &&
+      IsZeroSize(roundedRect.upper_left_corner_radius) &&
+      IsZeroSize(roundedRect.upper_right_corner_radius)) {
+    return CGPathCreateWithRect(FromFlutterRect(roundedRect.rect), nullptr);
+  }
+
   CGMutablePathRef path = CGPathCreateMutable();
 
   const auto& rect = roundedRect.rect;
@@ -306,7 +321,7 @@ typedef struct {
 } ClipRoundedRect;
 
 /// Returns the set of all rounded rect paths generated by clips in the mutations vector.
-NSMutableArray* RoundedRectClipsFromMutations(CGRect master_clip, const MutationVector& mutations) {
+NSMutableArray* ClipPathFromMutations(CGRect master_clip, const MutationVector& mutations) {
   std::vector<ClipRoundedRect> rounded_rects;
 
   CATransform3D transform = CATransform3DIdentity;
@@ -320,7 +335,17 @@ NSMutableArray* RoundedRectClipsFromMutations(CGRect master_clip, const Mutation
       case kFlutterPlatformViewMutationTypeTransformation:
         transform = CATransform3DConcat(ToCATransform3D(mutation.transformation), transform);
         break;
-      case kFlutterPlatformViewMutationTypeClipRect:
+      case kFlutterPlatformViewMutationTypeClipRect: {
+        CGAffineTransform affineTransform = CATransform3DGetAffineTransform(transform);
+        // Shearing or rotation requires path clipping.
+        if (!AffineTransformIsOnlyScaleOrTranslate(affineTransform)) {
+          rounded_rects.push_back(
+              {FlutterRoundedRect{mutation.clip_rect, FlutterSize{0, 0}, FlutterSize{0, 0},
+                                  FlutterSize{0, 0}, FlutterSize{0, 0}},
+               affineTransform});
+        }
+        break;
+      }
       case kFlutterPlatformViewMutationTypeOpacity:
         break;
     }
@@ -328,14 +353,18 @@ NSMutableArray* RoundedRectClipsFromMutations(CGRect master_clip, const Mutation
 
   NSMutableArray* paths = [NSMutableArray array];
   for (const auto& r : rounded_rects) {
-    CGAffineTransform inverse = CGAffineTransformInvert(r.transform);
-    // Transform master clip to clip rect coordinates and check if this view intersects one of the
-    // corners, which means we need to use path clipping.
-    CGRect localMasterClip = CGRectApplyAffineTransform(master_clip, inverse);
+    bool requiresPath = !AffineTransformIsOnlyScaleOrTranslate(r.transform);
+    if (!requiresPath) {
+      CGAffineTransform inverse = CGAffineTransformInvert(r.transform);
+      // Transform master clip to clip rect coordinates and check if this view intersects one of the
+      // corners, which means we need to use path clipping.
+      CGRect localMasterClip = CGRectApplyAffineTransform(master_clip, inverse);
+      requiresPath = RoundRectCornerIntersects(r.rrect, ToFlutterRect(localMasterClip));
+    }
 
     // Only clip to rounded rectangle path if the view intersects some of the round corners. If
     // not, clipping to masterClip is enough.
-    if (RoundRectCornerIntersects(r.rrect, ToFlutterRect(localMasterClip))) {
+    if (requiresPath) {
       CGPathRef path = PathFromRoundedRect(r.rrect);
       CGPathRef transformedPath = CGPathCreateCopyByTransformingPath(path, &r.transform);
       [paths addObject:(__bridge id)transformedPath];
@@ -481,7 +510,7 @@ NSMutableArray* RoundedRectClipsFromMutations(CGRect master_clip, const Mutation
   self.hidden = NO;
 
   /// Paths in global logical coordinates that need to be clipped to.
-  NSMutableArray* paths = RoundedRectClipsFromMutations(masterClip, mutations);
+  NSMutableArray* paths = ClipPathFromMutations(masterClip, mutations);
   [self updatePathClipViewsWithPaths:paths];
 
   /// Update PlatformViewContainer, PlatformView, and apply transforms and axis-aligned clip rect.

engine/src/flutter/shell/platform/darwin/macos/framework/Source/FlutterMutatorViewTest.mm

@@ -13,6 +13,8 @@
 
 @end
 
+static constexpr float kMaxErr = 1e-10;
+
 namespace {
 void ApplyFlutterLayer(FlutterMutatorView* view,
                        FlutterSize size,
@@ -45,8 +47,6 @@ void ApplyFlutterLayer(FlutterMutatorView* view,
 // In order to avoid architecture-specific floating point differences we don't check for exact
 // equality using, for example, CATransform3DEqualToTransform.
 void ExpectTransform3DEqual(const CATransform3D& t, const CATransform3D& u) {
-  constexpr float kMaxErr = 1e-10;
-
   EXPECT_NEAR(t.m11, u.m11, kMaxErr);
   EXPECT_NEAR(t.m12, u.m12, kMaxErr);
   EXPECT_NEAR(t.m13, u.m13, kMaxErr);
@@ -340,6 +340,42 @@ TEST(FlutterMutatorViewTest, ViewsSetIsFlipped) {
   EXPECT_TRUE(mutatorView.platformViewContainer.isFlipped);
 }
 
+TEST(FlutterMutatorViewTest, RectsClipsToPathWhenRotated) {
+  NSView* platformView = [[NSView alloc] init];
+  FlutterMutatorView* mutatorView = [[FlutterMutatorView alloc] initWithPlatformView:platformView];
+  std::vector<FlutterPlatformViewMutation> mutations{
+      {
+          .type = kFlutterPlatformViewMutationTypeTransformation,
+          // Roation M_PI / 8
+          .transformation =
+              FlutterTransformation{
+                  .scaleX = 0.9238795325112867,
+                  .skewX = -0.3826834323650898,
+                  .skewY = 0.3826834323650898,
+                  .scaleY = 0.9238795325112867,
+              },
+      },
+      {
+          .type = kFlutterPlatformViewMutationTypeClipRect,
+          .clip_rect = FlutterRect{110, 60, 150, 150},
+      },
+      {
+          .type = kFlutterPlatformViewMutationTypeTransformation,
+          .transformation =
+              FlutterTransformation{
+                  .scaleX = 1,
+                  .transX = 100,
+                  .scaleY = 1,
+                  .transY = 50,
+              },
+      },
+  };
+  ApplyFlutterLayer(mutatorView, FlutterSize{30, 20}, mutations);
+  EXPECT_EQ(mutatorView.pathClipViews.count, 1ul);
+  EXPECT_NEAR(mutatorView.platformViewContainer.frame.size.width, 35.370054622640396, kMaxErr);
+  EXPECT_NEAR(mutatorView.platformViewContainer.frame.size.height, 29.958093621178421, kMaxErr);
+}
+
 TEST(FlutterMutatorViewTest, RoundRectClipsToPath) {
   NSView* platformView = [[NSView alloc] init];
   FlutterMutatorView* mutatorView = [[FlutterMutatorView alloc] initWithPlatformView:platformView];