DEV/flutter_flutter
1
0
Fork 0
mirror of https://github.com/flutter/flutter.git synced 2026-02-20 02:29:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

Reland "[metal] Darwin unified external metal textures (#24157)" (flutter/engine#24244)

Browse Source
This commit is contained in:
Kaushik Iska 2021-02-09 10:45:16 -06:00 committed by GitHub
parent 5bb887be6d
commit 1deabe1dec
12 changed files with 370 additions and 268 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
engine/src/flutter/ci/licenses_golden/licenses_flutter
View File

@ -969,6 +969,8 @@ FILE: ../../../flutter/shell/platform/darwin/common/framework/Source/flutter_cod
FILE: ../../../flutter/shell/platform/darwin/common/framework/Source/flutter_standard_codec_unittest.mm
FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetal.h
FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetal.mm
FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h
FILE: ../../../flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm
FILE: ../../../flutter/shell/platform/darwin/ios/framework/Flutter.podspec
FILE: ../../../flutter/shell/platform/darwin/ios/framework/Headers/Flutter.h
FILE: ../../../flutter/shell/platform/darwin/ios/framework/Headers/FlutterAppDelegate.h

2
engine/src/flutter/common/config.gni
View File

@ -62,4 +62,6 @@ if (is_ios || is_mac) {
"-Werror=undeclared-selector",
]
flutter_cflags_objcc = flutter_cflags_objc
flutter_cflags_objc_arc = flutter_cflags_objc + [ "-fobjc-arc" ]
flutter_cflags_objcc_arc = flutter_cflags_objc_arc
}

4
engine/src/flutter/common/graphics/texture.h
View File

@ -18,8 +18,8 @@ namespace flutter {
class Texture {
public:
Texture(int64_t id); // Called from UI or raster thread.
virtual ~Texture(); // Called from raster thread.
explicit Texture(int64_t id); // Called from UI or raster thread.
virtual ~Texture(); // Called from raster thread.
// Called from raster thread.
virtual void Paint(SkCanvas& canvas,

9
engine/src/flutter/shell/platform/darwin/graphics/BUILD.gn
View File

@ -7,19 +7,24 @@ assert(is_ios || is_mac)
import("//flutter/common/config.gni")
source_set("graphics") {
cflags_objc = flutter_cflags_objc
cflags_objcc = flutter_cflags_objcc
cflags_objc = flutter_cflags_objc_arc
cflags_objcc = flutter_cflags_objcc_arc
sources = [
"FlutterDarwinContextMetal.h",
"FlutterDarwinContextMetal.mm",
"FlutterDarwinExternalTextureMetal.h",
"FlutterDarwinExternalTextureMetal.mm",
]
deps = [
"//flutter/common/graphics",
"//flutter/fml",
"//flutter/shell/platform/darwin/common:framework_shared",
]
libs = [ "CoreVideo.framework" ]
public_deps = [ "//third_party/skia" ]
public_configs = [ "//flutter:config" ]

15
engine/src/flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetal.h
View File

@ -5,9 +5,12 @@
#ifndef SHELL_PLATFORM_DARWIN_GRAPHICS_DARWIN_CONTEXT_METAL_H_
#define SHELL_PLATFORM_DARWIN_GRAPHICS_DARWIN_CONTEXT_METAL_H_
#import <CoreVideo/CVMetalTextureCache.h>
#import <Foundation/Foundation.h>
#import <Metal/Metal.h>
#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterTexture.h"
#import "flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h"
#include "third_party/skia/include/gpu/GrDirectContext.h"
NS_ASSUME_NONNULL_BEGIN
@ -29,6 +32,13 @@ NS_ASSUME_NONNULL_BEGIN
- (instancetype)initWithMTLDevice:(id<MTLDevice>)device
commandQueue:(id<MTLCommandQueue>)commandQueue;
/**
* Creates an external texture with the specified ID and contents.
*/
- (FlutterDarwinExternalTextureMetal*)
createExternalTextureWithIdentifier:(int64_t)textureID
texture:(NSObject<FlutterTexture>*)texture;
/**
* MTLDevice that is backing this context.s
*/
@ -50,6 +60,11 @@ NS_ASSUME_NONNULL_BEGIN
*/
@property(nonatomic, readonly) sk_sp<GrDirectContext> resourceContext;
/*
* Texture cache for external textures.
*/
@property(nonatomic, readonly) CVMetalTextureCacheRef textureCache;
@end
NS_ASSUME_NONNULL_END

49
engine/src/flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetal.mm
View File

@ -6,8 +6,11 @@
#include "flutter/common/graphics/persistent_cache.h"
#include "flutter/fml/logging.h"
#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterMacros.h"
#include "third_party/skia/include/gpu/GrContextOptions.h"
FLUTTER_ASSERT_ARC
static GrContextOptions CreateMetalGrContextOptions() {
GrContextOptions options = {};
if (flutter::PersistentCache::cache_sksl()) {
@ -33,7 +36,6 @@ static GrContextOptions CreateMetalGrContextOptions() {
if (!_device) {
FML_DLOG(ERROR) << "Could not acquire Metal device.";
[self release];
return nil;
}
@ -41,24 +43,27 @@ static GrContextOptions CreateMetalGrContextOptions() {
if (!_commandQueue) {
FML_DLOG(ERROR) << "Could not create Metal command queue.";
[self release];
return nil;
}
[_commandQueue setLabel:@"Flutter Main Queue"];
auto contextOptions = CreateMetalGrContextOptions();
CVReturn cvReturn = CVMetalTextureCacheCreate(kCFAllocatorDefault, // allocator
nil, // cache attributes (nil default)
_device, // metal device
nil, // texture attributes (nil default)
&_textureCache // [out] cache
);
if (cvReturn != kCVReturnSuccess) {
FML_DLOG(ERROR) << "Could not create Metal texture cache.";
return nil;
}
// Skia expect arguments to `MakeMetal` transfer ownership of the reference in for release later
// when the GrDirectContext is collected.
_mainContext =
GrDirectContext::MakeMetal([_device retain], [_commandQueue retain], contextOptions);
_resourceContext =
GrDirectContext::MakeMetal([_device retain], [_commandQueue retain], contextOptions);
_mainContext = [self createGrContext];
_resourceContext = [self createGrContext];
if (!_mainContext || !_resourceContext) {
FML_DLOG(ERROR) << "Could not create Skia Metal contexts.";
[self release];
return nil;
}
@ -67,4 +72,28 @@ static GrContextOptions CreateMetalGrContextOptions() {
return self;
}
- (sk_sp<GrDirectContext>)createGrContext {
auto contextOptions = CreateMetalGrContextOptions();
id<MTLDevice> device = _device;
id<MTLCommandQueue> commandQueue = _commandQueue;
// Skia expect arguments to `MakeMetal` transfer ownership of the reference in for release later
// when the GrDirectContext is collected.
return GrDirectContext::MakeMetal((__bridge_retained void*)device,
(__bridge_retained void*)commandQueue, contextOptions);
}
- (void)dealloc {
if (_textureCache) {
CFRelease(_textureCache);
}
}
- (FlutterDarwinExternalTextureMetal*)
createExternalTextureWithIdentifier:(int64_t)textureID
texture:(NSObject<FlutterTexture>*)texture {
return [[FlutterDarwinExternalTextureMetal alloc] initWithTextureCache:_textureCache
textureID:textureID
texture:texture];
}
@end

34
engine/src/flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h Normal file
View File

@ -0,0 +1,34 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#import <Foundation/Foundation.h>
#import <Metal/Metal.h>
#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterTexture.h"
#include "third_party/skia/include/core/SkCanvas.h"
#include "third_party/skia/include/gpu/GrDirectContext.h"
@interface FlutterDarwinExternalTextureMetal : NSObject
- (nullable instancetype)initWithTextureCache:(nonnull CVMetalTextureCacheRef)textureCache
textureID:(int64_t)textureID
texture:(nonnull NSObject<FlutterTexture>*)texture;
- (void)paint:(SkCanvas&)canvas
bounds:(const SkRect&)bounds
freeze:(BOOL)freeze
grContext:(nonnull GrDirectContext*)grContext
sampling:(const SkSamplingOptions&)sampling;
- (void)onGrContextCreated;
- (void)onGrContextDestroyed;
- (void)markNewFrameAvailable;
- (void)onTextureUnregistered;
@property(nonatomic, readonly) int64_t textureID;
@end

245
engine/src/flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm Normal file
View File

@ -0,0 +1,245 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#import "flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h"
#include "flutter/fml/logging.h"
#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterMacros.h"
#include "third_party/skia/include/core/SkYUVAInfo.h"
#include "third_party/skia/include/gpu/GrBackendSurface.h"
#include "third_party/skia/include/gpu/GrDirectContext.h"
#include "third_party/skia/include/gpu/GrYUVABackendTextures.h"
#include "third_party/skia/include/gpu/mtl/GrMtlTypes.h"
FLUTTER_ASSERT_ARC
@implementation FlutterDarwinExternalTextureMetal {
CVMetalTextureCacheRef _textureCache;
NSObject<FlutterTexture>* _externalTexture;
BOOL _textureFrameAvailable;
sk_sp<SkImage> _externalImage;
CVPixelBufferRef _lastPixelBuffer;
OSType _pixelFormat;
}
- (instancetype)initWithTextureCache:(nonnull CVMetalTextureCacheRef)textureCache
textureID:(int64_t)textureID
texture:(NSObject<FlutterTexture>*)texture {
if (self = [super init]) {
_textureCache = textureCache;
CFRetain(_textureCache);
_textureID = textureID;
_externalTexture = texture;
return self;
}
return nil;
}
- (void)dealloc {
CVPixelBufferRelease(_lastPixelBuffer);
if (_textureCache) {
CFRelease(_textureCache);
}
}
- (void)paint:(SkCanvas&)canvas
bounds:(const SkRect&)bounds
freeze:(BOOL)freeze
grContext:(nonnull GrDirectContext*)grContext
sampling:(const SkSamplingOptions&)sampling {
const bool needsUpdatedTexture = (!freeze && _textureFrameAvailable) || !_externalImage;
if (needsUpdatedTexture) {
CVPixelBufferRef pixelBuffer = [_externalTexture copyPixelBuffer];
if (!pixelBuffer) {
pixelBuffer = _lastPixelBuffer;
} else {
CVPixelBufferRetain(pixelBuffer);
_pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer);
}
// If the application told us there was a texture frame available but did not provide one when
// asked for it, reuse the previous texture but make sure to ask again the next time around.
sk_sp<SkImage> image = [self wrapExternalPixelBuffer:pixelBuffer grContext:grContext];
if (image) {
_externalImage = image;
_textureFrameAvailable = false;
CVPixelBufferRelease(_lastPixelBuffer);
_lastPixelBuffer = pixelBuffer;
}
}
if (_externalImage) {
canvas.drawImageRect(_externalImage, // image
SkRect::Make(_externalImage->bounds()), // source rect
bounds, // destination rect
sampling, // sampling
nullptr, // paint
SkCanvas::SrcRectConstraint::kFast_SrcRectConstraint // constraint
);
}
}
- (void)onGrContextCreated {
// External images in this backend have no thread affinity and are not tied to the context in any
// way. Instead, they are tied to the Metal device which is associated with the cache already and
// is consistent throughout the shell run.
}
- (void)onGrContextDestroyed {
// The image must be reset because it is tied to the onscreen context. But the pixel buffer that
// created the image is still around. In case of context reacquisition, that last pixel
// buffer will be used to materialize the image in case the application fails to provide a new
// one.
_externalImage.reset();
CVMetalTextureCacheFlush(_textureCache, // cache
0 // options (must be zero)
);
}
- (void)markNewFrameAvailable {
_textureFrameAvailable = YES;
}
- (void)onTextureUnregistered {
if ([_externalTexture respondsToSelector:@selector(onTextureUnregistered:)]) {
[_externalTexture onTextureUnregistered:_externalTexture];
}
}
#pragma mark - External texture skia wrapper methods.
- (sk_sp<SkImage>)wrapExternalPixelBuffer:(CVPixelBufferRef)pixelBuffer
grContext:(GrDirectContext*)grContext {
if (!pixelBuffer) {
return nullptr;
}
sk_sp<SkImage> image = nullptr;
if (_pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ||
_pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
image = [self wrapNV12ExternalPixelBuffer:pixelBuffer grContext:grContext];
} else {
image = [self wrapRGBAExternalPixelBuffer:pixelBuffer grContext:grContext];
}
if (!image) {
FML_DLOG(ERROR) << "Could not wrap Metal texture as a Skia image.";
}
return image;
}
- (sk_sp<SkImage>)wrapNV12ExternalPixelBuffer:(CVPixelBufferRef)pixelBuffer
grContext:(GrDirectContext*)grContext {
SkISize textureSize =
SkISize::Make(CVPixelBufferGetWidth(pixelBuffer), CVPixelBufferGetHeight(pixelBuffer));
CVMetalTextureRef yMetalTexture = nullptr;
{
CVReturn cvReturn =
CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault,
/*textureCache=*/_textureCache,
/*sourceImage=*/pixelBuffer,
/*textureAttributes=*/nullptr,
/*pixelFormat=*/MTLPixelFormatR8Unorm,
/*width=*/textureSize.width(),
/*height=*/textureSize.height(),
/*planeIndex=*/0u,
/*texture=*/&yMetalTexture);
if (cvReturn != kCVReturnSuccess) {
FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cvReturn;
return nullptr;
}
}
CVMetalTextureRef uvMetalTexture = nullptr;
{
CVReturn cvReturn =
CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault,
/*textureCache=*/_textureCache,
/*sourceImage=*/pixelBuffer,
/*textureAttributes=*/nullptr,
/*pixelFormat=*/MTLPixelFormatRG8Unorm,
/*width=*/textureSize.width() / 2,
/*height=*/textureSize.height() / 2,
/*planeIndex=*/1u,
/*texture=*/&uvMetalTexture);
if (cvReturn != kCVReturnSuccess) {
FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cvReturn;
return nullptr;
}
}
GrMtlTextureInfo ySkiaTextureInfo;
ySkiaTextureInfo.fTexture = sk_cf_obj<const void*>{
(__bridge_retained const void*)CVMetalTextureGetTexture(yMetalTexture)};
GrBackendTexture skiaBackendTextures[2];
skiaBackendTextures[0] = GrBackendTexture(/*width=*/textureSize.width(),
/*height=*/textureSize.height(),
/*mipMapped=*/GrMipMapped ::kNo,
/*textureInfo=*/ySkiaTextureInfo);
GrMtlTextureInfo uvSkiaTextureInfo;
uvSkiaTextureInfo.fTexture = sk_cf_obj<const void*>{
(__bridge_retained const void*)CVMetalTextureGetTexture(uvMetalTexture)};
skiaBackendTextures[1] = GrBackendTexture(/*width=*/textureSize.width(),
/*height=*/textureSize.height(),
/*mipMapped=*/GrMipMapped ::kNo,
/*textureInfo=*/uvSkiaTextureInfo);
SkYUVAInfo yuvaInfo(skiaBackendTextures[0].dimensions(), SkYUVAInfo::PlaneConfig::kY_UV,
SkYUVAInfo::Subsampling::k444, kRec601_SkYUVColorSpace);
GrYUVABackendTextures yuvaBackendTextures(yuvaInfo, skiaBackendTextures,
kTopLeft_GrSurfaceOrigin);
sk_sp<SkImage> image =
SkImage::MakeFromYUVATextures(grContext, yuvaBackendTextures, /*imageColorSpace=*/nullptr,
/*releaseProc*/ nullptr, /*releaseContext*/ nullptr);
return image;
}
- (sk_sp<SkImage>)wrapRGBAExternalPixelBuffer:(CVPixelBufferRef)pixelBuffer
grContext:(GrDirectContext*)grContext {
SkISize textureSize =
SkISize::Make(CVPixelBufferGetWidth(pixelBuffer), CVPixelBufferGetHeight(pixelBuffer));
CVMetalTextureRef metalTexture = nullptr;
CVReturn cvReturn =
CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault,
/*textureCache=*/_textureCache,
/*sourceImage=*/pixelBuffer,
/*textureAttributes=*/nullptr,
/*pixelFormat=*/MTLPixelFormatBGRA8Unorm,
/*width=*/textureSize.width(),
/*height=*/textureSize.height(),
/*planeIndex=*/0u,
/*texture=*/&metalTexture);
if (cvReturn != kCVReturnSuccess) {
FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cvReturn;
return nullptr;
}
GrMtlTextureInfo skiaTextureInfo;
skiaTextureInfo.fTexture =
sk_cf_obj<const void*>{(__bridge_retained const void*)CVMetalTextureGetTexture(metalTexture)};
GrBackendTexture skiaBackendTexture(/*width=*/textureSize.width(),
/*height=*/textureSize.height(),
/*mipMapped=*/GrMipMapped ::kNo,
/*textureInfo=*/skiaTextureInfo);
sk_sp<SkImage> image =
SkImage::MakeFromTexture(grContext, skiaBackendTexture, kTopLeft_GrSurfaceOrigin,
kBGRA_8888_SkColorType, kPremul_SkAlphaType,
/*imageColorSpace=*/nullptr, /*releaseProc*/ nullptr,
/*releaseContext*/ nullptr
);
return image;
}
@end

5
engine/src/flutter/shell/platform/darwin/ios/ios_context_metal.mm
View File

@ -65,7 +65,10 @@ std::unique_ptr<GLContextResult> IOSContextMetal::MakeCurrent() {
std::unique_ptr<Texture> IOSContextMetal::CreateExternalTexture(
int64_t texture_id,
fml::scoped_nsobject<NSObject<FlutterTexture>> texture) {
return std::make_unique<IOSExternalTextureMetal>(texture_id, texture_cache_, std::move(texture));
return std::make_unique<IOSExternalTextureMetal>(
fml::scoped_nsobject<FlutterDarwinExternalTextureMetal>{
[[darwin_context_metal_ createExternalTextureWithIdentifier:texture_id
texture:texture] retain]});
}
} // namespace flutter

29
engine/src/flutter/shell/platform/darwin/ios/ios_external_texture_metal.h
View File

@ -5,35 +5,25 @@
#ifndef FLUTTER_SHELL_PLATFORM_DARWIN_IOS_IOS_EXTERNAL_TEXTURE_METAL_H_
#define FLUTTER_SHELL_PLATFORM_DARWIN_IOS_IOS_EXTERNAL_TEXTURE_METAL_H_
#include <atomic>
#import <CoreVideo/CoreVideo.h>
#include "flutter/common/graphics/texture.h"
#include "flutter/fml/macros.h"
#include "flutter/fml/platform/darwin/cf_utils.h"
#include "flutter/fml/platform/darwin/scoped_nsobject.h"
#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterTexture.h"
#include "third_party/skia/include/core/SkImage.h"
#import "flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h"
namespace flutter {
class IOSExternalTextureMetal final : public Texture {
public:
IOSExternalTextureMetal(int64_t texture_id,
fml::CFRef<CVMetalTextureCacheRef> texture_cache,
fml::scoped_nsobject<NSObject<FlutterTexture>> external_texture);
explicit IOSExternalTextureMetal(
fml::scoped_nsobject<FlutterDarwinExternalTextureMetal>
darwin_external_texture_metal);
// |Texture|
~IOSExternalTextureMetal();
private:
fml::CFRef<CVMetalTextureCacheRef> texture_cache_;
fml::scoped_nsobject<NSObject<FlutterTexture>> external_texture_;
std::atomic_bool texture_frame_available_;
fml::CFRef<CVPixelBufferRef> last_pixel_buffer_;
sk_sp<SkImage> external_image_;
OSType pixel_format_ = 0;
fml::scoped_nsobject<FlutterDarwinExternalTextureMetal>
darwin_external_texture_metal_;
// |Texture|
void Paint(SkCanvas& canvas,
@ -54,13 +44,6 @@ class IOSExternalTextureMetal final : public Texture {
// |Texture|
void OnTextureUnregistered() override;
sk_sp<SkImage> WrapExternalPixelBuffer(fml::CFRef<CVPixelBufferRef> pixel_buffer,
GrDirectContext* context) const;
sk_sp<SkImage> WrapRGBAExternalPixelBuffer(fml::CFRef<CVPixelBufferRef> pixel_buffer,
GrDirectContext* context) const;
sk_sp<SkImage> WrapNV12ExternalPixelBuffer(fml::CFRef<CVPixelBufferRef> pixel_buffer,
GrDirectContext* context) const;
FML_DISALLOW_COPY_AND_ASSIGN(IOSExternalTextureMetal);
};

240
engine/src/flutter/shell/platform/darwin/ios/ios_external_texture_metal.mm
View File

@ -4,25 +4,12 @@
#import "flutter/shell/platform/darwin/ios/ios_external_texture_metal.h"
#include "flutter/fml/logging.h"
#include "third_party/skia/include/core/SkYUVAInfo.h"
#include "third_party/skia/include/gpu/GrBackendSurface.h"
#include "third_party/skia/include/gpu/GrDirectContext.h"
#include "third_party/skia/include/gpu/GrYUVABackendTextures.h"
#include "third_party/skia/include/gpu/mtl/GrMtlTypes.h"
namespace flutter {
IOSExternalTextureMetal::IOSExternalTextureMetal(
int64_t texture_id,
fml::CFRef<CVMetalTextureCacheRef> texture_cache,
fml::scoped_nsobject<NSObject<FlutterTexture>> external_texture)
: Texture(texture_id),
texture_cache_(std::move(texture_cache)),
external_texture_(std::move(external_texture)) {
FML_DCHECK(texture_cache_);
FML_DCHECK(external_texture_);
}
fml::scoped_nsobject<FlutterDarwinExternalTextureMetal> darwin_external_texture_metal)
: Texture([darwin_external_texture_metal textureID]),
darwin_external_texture_metal_(darwin_external_texture_metal) {}
IOSExternalTextureMetal::~IOSExternalTextureMetal() = default;
@ -31,230 +18,27 @@ void IOSExternalTextureMetal::Paint(SkCanvas& canvas,
bool freeze,
GrDirectContext* context,
const SkSamplingOptions& sampling) {
const bool needs_updated_texture = (!freeze && texture_frame_available_) || !external_image_;
if (needs_updated_texture) {
auto pixel_buffer = fml::CFRef<CVPixelBufferRef>([external_texture_ copyPixelBuffer]);
if (!pixel_buffer) {
pixel_buffer = std::move(last_pixel_buffer_);
} else {
pixel_format_ = CVPixelBufferGetPixelFormatType(pixel_buffer);
}
// If the application told us there was a texture frame available but did not provide one when
// asked for it, reuse the previous texture but make sure to ask again the next time around.
if (auto wrapped_texture = WrapExternalPixelBuffer(pixel_buffer, context)) {
external_image_ = wrapped_texture;
texture_frame_available_ = false;
last_pixel_buffer_ = std::move(pixel_buffer);
}
}
if (external_image_) {
canvas.drawImageRect(external_image_, // image
SkRect::Make(external_image_->bounds()), // source rect
bounds, // destination rect
sampling,
nullptr, // paint
SkCanvas::SrcRectConstraint::kFast_SrcRectConstraint // constraint
);
}
}
sk_sp<SkImage> IOSExternalTextureMetal::WrapExternalPixelBuffer(
fml::CFRef<CVPixelBufferRef> pixel_buffer,
GrDirectContext* context) const {
if (!pixel_buffer) {
return nullptr;
}
sk_sp<SkImage> image = nullptr;
if (pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ||
pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
image = WrapNV12ExternalPixelBuffer(pixel_buffer, context);
} else {
image = WrapRGBAExternalPixelBuffer(pixel_buffer, context);
}
if (!image) {
FML_DLOG(ERROR) << "Could not wrap Metal texture as a Skia image.";
}
return image;
}
sk_sp<SkImage> IOSExternalTextureMetal::WrapNV12ExternalPixelBuffer(
fml::CFRef<CVPixelBufferRef> pixel_buffer,
GrDirectContext* context) const {
auto texture_size =
SkISize::Make(CVPixelBufferGetWidth(pixel_buffer), CVPixelBufferGetHeight(pixel_buffer));
CVMetalTextureRef y_metal_texture_raw = nullptr;
{
auto cv_return =
CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault,
/*textureCache=*/texture_cache_,
/*sourceImage=*/pixel_buffer,
/*textureAttributes=*/nullptr,
/*pixelFormat=*/MTLPixelFormatR8Unorm,
/*width=*/texture_size.width(),
/*height=*/texture_size.height(),
/*planeIndex=*/0u,
/*texture=*/&y_metal_texture_raw);
if (cv_return != kCVReturnSuccess) {
FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cv_return;
return nullptr;
}
}
CVMetalTextureRef uv_metal_texture_raw = nullptr;
{
auto cv_return =
CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault,
/*textureCache=*/texture_cache_,
/*sourceImage=*/pixel_buffer,
/*textureAttributes=*/nullptr,
/*pixelFormat=*/MTLPixelFormatRG8Unorm,
/*width=*/texture_size.width() / 2,
/*height=*/texture_size.height() / 2,
/*planeIndex=*/1u,
/*texture=*/&uv_metal_texture_raw);
if (cv_return != kCVReturnSuccess) {
FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cv_return;
return nullptr;
}
}
fml::CFRef<CVMetalTextureRef> y_metal_texture(y_metal_texture_raw);
GrMtlTextureInfo y_skia_texture_info;
y_skia_texture_info.fTexture = sk_cf_obj<const void*>{
[reinterpret_cast<NSObject*>(CVMetalTextureGetTexture(y_metal_texture)) retain]};
GrBackendTexture skia_backend_textures[2];
skia_backend_textures[0] = GrBackendTexture(/*width=*/texture_size.width(),
/*height=*/texture_size.height(),
/*mipMapped=*/GrMipMapped ::kNo,
/*textureInfo=*/y_skia_texture_info);
fml::CFRef<CVMetalTextureRef> uv_metal_texture(uv_metal_texture_raw);
GrMtlTextureInfo uv_skia_texture_info;
uv_skia_texture_info.fTexture = sk_cf_obj<const void*>{
[reinterpret_cast<NSObject*>(CVMetalTextureGetTexture(uv_metal_texture)) retain]};
skia_backend_textures[1] = GrBackendTexture(/*width=*/texture_size.width(),
/*height=*/texture_size.height(),
/*mipMapped=*/GrMipMapped ::kNo,
/*textureInfo=*/uv_skia_texture_info);
SkYUVAInfo yuva_info(skia_backend_textures[0].dimensions(), SkYUVAInfo::PlaneConfig::kY_UV,
SkYUVAInfo::Subsampling::k444, kRec601_SkYUVColorSpace);
GrYUVABackendTextures yuva_backend_textures(yuva_info, skia_backend_textures,
kTopLeft_GrSurfaceOrigin);
struct ImageCaptures {
fml::CFRef<CVPixelBufferRef> buffer;
fml::CFRef<CVMetalTextureRef> y_texture;
fml::CFRef<CVMetalTextureRef> uv_texture;
};
auto captures = std::make_unique<ImageCaptures>();
captures->buffer = std::move(pixel_buffer);
captures->y_texture = std::move(y_metal_texture);
captures->uv_texture = std::move(uv_metal_texture);
SkImage::TextureReleaseProc release_proc = [](SkImage::ReleaseContext release_context) {
auto captures = reinterpret_cast<ImageCaptures*>(release_context);
delete captures;
};
sk_sp<SkImage> image =
SkImage::MakeFromYUVATextures(context, yuva_backend_textures, /*imageColorSpace=*/nullptr,
release_proc, captures.release());
return image;
}
sk_sp<SkImage> IOSExternalTextureMetal::WrapRGBAExternalPixelBuffer(
fml::CFRef<CVPixelBufferRef> pixel_buffer,
GrDirectContext* context) const {
auto texture_size =
SkISize::Make(CVPixelBufferGetWidth(pixel_buffer), CVPixelBufferGetHeight(pixel_buffer));
CVMetalTextureRef metal_texture_raw = nullptr;
auto cv_return =
CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault,
/*textureCache=*/texture_cache_,
/*sourceImage=*/pixel_buffer,
/*textureAttributes=*/nullptr,
/*pixelFormat=*/MTLPixelFormatBGRA8Unorm,
/*width=*/texture_size.width(),
/*height=*/texture_size.height(),
/*planeIndex=*/0u,
/*texture=*/&metal_texture_raw);
if (cv_return != kCVReturnSuccess) {
FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cv_return;
return nullptr;
}
fml::CFRef<CVMetalTextureRef> metal_texture(metal_texture_raw);
GrMtlTextureInfo skia_texture_info;
skia_texture_info.fTexture = sk_cf_obj<const void*>{
[reinterpret_cast<NSObject*>(CVMetalTextureGetTexture(metal_texture)) retain]};
GrBackendTexture skia_backend_texture(/*width=*/texture_size.width(),
/*height=*/texture_size.height(),
/*mipMapped=*/GrMipMapped ::kNo,
/*textureInfo=*/skia_texture_info);
struct ImageCaptures {
fml::CFRef<CVPixelBufferRef> buffer;
fml::CFRef<CVMetalTextureRef> texture;
};
auto captures = std::make_unique<ImageCaptures>();
captures->buffer = std::move(pixel_buffer);
captures->texture = std::move(metal_texture);
SkImage::TextureReleaseProc release_proc = [](SkImage::ReleaseContext release_context) {
auto captures = reinterpret_cast<ImageCaptures*>(release_context);
delete captures;
};
auto image =
SkImage::MakeFromTexture(context, skia_backend_texture, kTopLeft_GrSurfaceOrigin,
kBGRA_8888_SkColorType, kPremul_SkAlphaType,
/*imageColorSpace=*/nullptr, release_proc, captures.release()
);
return image;
[darwin_external_texture_metal_ paint:canvas
bounds:bounds
freeze:freeze
grContext:context
sampling:sampling];
}
void IOSExternalTextureMetal::OnGrContextCreated() {
// External images in this backend have no thread affinity and are not tied to the context in any
// way. Instead, they are tied to the Metal device which is associated with the cache already and
// is consistent throughout the shell run.
[darwin_external_texture_metal_ onGrContextCreated];
}
void IOSExternalTextureMetal::OnGrContextDestroyed() {
// The image must be reset because it is tied to the onscreen context. But the pixel buffer that
// created the image is still around. In case of context reacquisition, that last pixel
// buffer will be used to materialize the image in case the application fails to provide a new
// one.
external_image_.reset();
CVMetalTextureCacheFlush(texture_cache_, // cache
0 // options (must be zero)
);
[darwin_external_texture_metal_ onGrContextDestroyed];
}
void IOSExternalTextureMetal::MarkNewFrameAvailable() {
texture_frame_available_ = true;
[darwin_external_texture_metal_ markNewFrameAvailable];
}
void IOSExternalTextureMetal::OnTextureUnregistered() {
if ([external_texture_ respondsToSelector:@selector(onTextureUnregistered:)]) {
[external_texture_ onTextureUnregistered:external_texture_];
}
[darwin_external_texture_metal_ onTextureUnregistered];
}
} // namespace flutter

4
engine/src/flutter/shell/platform/darwin/macos/BUILD.gn
View File

@ -115,7 +115,7 @@ source_set("flutter_framework_source") {
"FLUTTER_ENGINE_NO_PROTOTYPES",
]
cflags_objcc = [ "-fobjc-arc" ]
cflags_objcc = flutter_cflags_objcc_arc
libs = [
"Cocoa.framework",
@ -161,7 +161,7 @@ executable("flutter_desktop_darwin_unittests") {
sources += [ "framework/Source/FlutterOpenGLRendererTest.mm" ]
}
cflags_objcc = [ "-fobjc-arc" ]
cflags_objcc = flutter_cflags_objcc_arc
ldflags = [ "-ObjC" ]