DEV/flutter_flutter
1
0
Fork 0
mirror of https://github.com/flutter/flutter.git synced 2026-02-20 02:29:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

iOS external texture supports rendering NV12 pixelbuffer (flutter/engine#20082)

Browse Source
This commit is contained in:
DongXu 2020-08-04 02:20:45 +08:00 committed by GitHub
parent 59b75c2dcd
commit 3944bf4015
4 changed files with 248 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
engine/src/flutter/shell/platform/darwin/ios/ios_external_texture_gl.h
View File

@ -25,6 +25,9 @@ class IOSExternalTextureGL final : public Texture {
fml::CFRef<CVOpenGLESTextureCacheRef> cache_ref_;
fml::CFRef<CVOpenGLESTextureRef> texture_ref_;
fml::CFRef<CVPixelBufferRef> buffer_ref_;
OSType pixel_format_ = 0;
fml::CFRef<CVOpenGLESTextureRef> y_texture_ref_;
fml::CFRef<CVOpenGLESTextureRef> uv_texture_ref_;
// |Texture|
void Paint(SkCanvas& canvas,
@ -51,6 +54,16 @@ class IOSExternalTextureGL final : public Texture {
bool NeedUpdateTexture(bool freeze);
bool IsTexturesAvailable() const;
void CreateYUVTexturesFromPixelBuffer();
void CreateRGBATextureFromPixelBuffer();
sk_sp<SkImage> CreateImageFromYUVTextures(GrContext* context, const SkRect& bounds);
sk_sp<SkImage> CreateImageFromRGBATexture(GrContext* context, const SkRect& bounds);
FML_DISALLOW_COPY_AND_ASSIGN(IOSExternalTextureGL);
};

108
engine/src/flutter/shell/platform/darwin/ios/ios_external_texture_gl.mm
View File

@ -10,8 +10,10 @@
#include "flutter/shell/platform/darwin/ios/framework/Source/vsync_waiter_ios.h"
#include "third_party/skia/include/core/SkSurface.h"
#include "third_party/skia/include/core/SkYUVAIndex.h"
#include "third_party/skia/include/gpu/GrBackendSurface.h"
#include "third_party/skia/include/gpu/GrDirectContext.h"
#include "third_party/skia/src/gpu/gl/GrGLDefines.h"
namespace flutter {
@ -42,10 +44,19 @@ void IOSExternalTextureGL::CreateTextureFromPixelBuffer() {
if (buffer_ref_ == nullptr) {
return;
}
if (pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ||
pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
CreateYUVTexturesFromPixelBuffer();
} else {
CreateRGBATextureFromPixelBuffer();
}
}
void IOSExternalTextureGL::CreateRGBATextureFromPixelBuffer() {
CVOpenGLESTextureRef texture;
CVReturn err = CVOpenGLESTextureCacheCreateTextureFromImage(
kCFAllocatorDefault, cache_ref_, buffer_ref_, nullptr, GL_TEXTURE_2D, GL_RGBA,
static_cast<int>(CVPixelBufferGetWidth(buffer_ref_)),
kCFAllocatorDefault, cache_ref_, buffer_ref_, /*textureAttributes=*/nullptr, GL_TEXTURE_2D,
GL_RGBA, static_cast<int>(CVPixelBufferGetWidth(buffer_ref_)),
static_cast<int>(CVPixelBufferGetHeight(buffer_ref_)), GL_BGRA, GL_UNSIGNED_BYTE, 0,
&texture);
if (err != noErr) {
@ -55,10 +66,83 @@ void IOSExternalTextureGL::CreateTextureFromPixelBuffer() {
}
}
void IOSExternalTextureGL::CreateYUVTexturesFromPixelBuffer() {
size_t width = CVPixelBufferGetWidth(buffer_ref_);
size_t height = CVPixelBufferGetHeight(buffer_ref_);
{
CVOpenGLESTextureRef yTexture;
CVReturn err = CVOpenGLESTextureCacheCreateTextureFromImage(
kCFAllocatorDefault, cache_ref_, buffer_ref_, /*textureAttributes=*/nullptr, GL_TEXTURE_2D,
GL_LUMINANCE, width, height, GL_LUMINANCE, GL_UNSIGNED_BYTE, 0, &yTexture);
if (err != noErr) {
FML_DCHECK(yTexture) << "Could not create texture from pixel buffer: " << err;
} else {
y_texture_ref_.Reset(yTexture);
}
}
{
CVOpenGLESTextureRef uvTexture;
CVReturn err = CVOpenGLESTextureCacheCreateTextureFromImage(
kCFAllocatorDefault, cache_ref_, buffer_ref_, /*textureAttributes=*/nullptr, GL_TEXTURE_2D,
GL_LUMINANCE_ALPHA, width / 2, height / 2, GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, 1,
&uvTexture);
if (err != noErr) {
FML_DCHECK(uvTexture) << "Could not create texture from pixel buffer: " << err;
} else {
uv_texture_ref_.Reset(uvTexture);
}
}
}
sk_sp<SkImage> IOSExternalTextureGL::CreateImageFromRGBATexture(GrContext* context,
const SkRect& bounds) {
GrGLTextureInfo textureInfo = {CVOpenGLESTextureGetTarget(texture_ref_),
CVOpenGLESTextureGetName(texture_ref_), GL_RGBA8_OES};
GrBackendTexture backendTexture(bounds.width(), bounds.height(), GrMipMapped::kNo, textureInfo);
sk_sp<SkImage> image = SkImage::MakeFromTexture(context, backendTexture, kTopLeft_GrSurfaceOrigin,
kRGBA_8888_SkColorType, kPremul_SkAlphaType,
/*imageColorSpace=*/nullptr);
return image;
}
sk_sp<SkImage> IOSExternalTextureGL::CreateImageFromYUVTextures(GrContext* context,
const SkRect& bounds) {
GrGLTextureInfo yTextureInfo = {CVOpenGLESTextureGetTarget(y_texture_ref_),
CVOpenGLESTextureGetName(y_texture_ref_), GR_GL_LUMINANCE8};
GrBackendTexture yBackendTexture(bounds.width(), bounds.height(), GrMipMapped::kNo, yTextureInfo);
GrGLTextureInfo uvTextureInfo = {CVOpenGLESTextureGetTarget(uv_texture_ref_),
CVOpenGLESTextureGetName(uv_texture_ref_), GR_GL_RGBA8};
GrBackendTexture uvBackendTexture(bounds.width(), bounds.height(), GrMipMapped::kNo,
uvTextureInfo);
GrBackendTexture nv12TextureHandles[] = {yBackendTexture, uvBackendTexture};
SkYUVAIndex yuvaIndices[4] = {
SkYUVAIndex{0, SkColorChannel::kR}, // Read Y data from the red channel of the first texture
SkYUVAIndex{1, SkColorChannel::kR}, // Read U data from the red channel of the second texture
SkYUVAIndex{
1, SkColorChannel::kA}, // Read V data from the alpha channel of the second texture,
// normal NV12 data V should be taken from the green channel, but
// currently only the uv texture created by GL_LUMINANCE_ALPHA
// can be used, so the V value is taken from the alpha channel
SkYUVAIndex{-1, SkColorChannel::kA}}; //-1 means to omit the alpha data of YUVA
SkISize size{yBackendTexture.width(), yBackendTexture.height()};
sk_sp<SkImage> image = SkImage::MakeFromYUVATextures(
context, kRec601_SkYUVColorSpace, nv12TextureHandles, yuvaIndices, size,
kTopLeft_GrSurfaceOrigin, /*imageColorSpace=*/nullptr);
return image;
}
bool IOSExternalTextureGL::IsTexturesAvailable() const {
return ((pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ||
pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) &&
(y_texture_ref_ && uv_texture_ref_)) ||
(pixel_format_ == kCVPixelFormatType_32BGRA && texture_ref_);
}
bool IOSExternalTextureGL::NeedUpdateTexture(bool freeze) {
// Update texture if `texture_ref_` is reset to `nullptr` when GrContext
// is destroyed or new frame is ready.
return (!freeze && new_frame_ready_) || !texture_ref_;
return (!freeze && new_frame_ready_) || !IsTexturesAvailable();
}
void IOSExternalTextureGL::Paint(SkCanvas& canvas,
@ -71,19 +155,23 @@ void IOSExternalTextureGL::Paint(SkCanvas& canvas,
auto pixelBuffer = [external_texture_.get() copyPixelBuffer];
if (pixelBuffer) {
buffer_ref_.Reset(pixelBuffer);
pixel_format_ = CVPixelBufferGetPixelFormatType(buffer_ref_);
}
CreateTextureFromPixelBuffer();
new_frame_ready_ = false;
}
if (!texture_ref_) {
if (!IsTexturesAvailable()) {
return;
}
GrGLTextureInfo textureInfo = {CVOpenGLESTextureGetTarget(texture_ref_),
CVOpenGLESTextureGetName(texture_ref_), GL_RGBA8_OES};
GrBackendTexture backendTexture(bounds.width(), bounds.height(), GrMipMapped::kNo, textureInfo);
sk_sp<SkImage> image =
SkImage::MakeFromTexture(context, backendTexture, kTopLeft_GrSurfaceOrigin,
kRGBA_8888_SkColorType, kPremul_SkAlphaType, nullptr);
sk_sp<SkImage> image = nullptr;
if (pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ||
pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
image = CreateImageFromYUVTextures(context, bounds);
} else {
image = CreateImageFromRGBATexture(context, bounds);
}
FML_DCHECK(image) << "Failed to create SkImage from Texture.";
if (image) {
SkPaint paint;

5
engine/src/flutter/shell/platform/darwin/ios/ios_external_texture_metal.h
View File

@ -33,6 +33,7 @@ class IOSExternalTextureMetal final : public Texture {
std::atomic_bool texture_frame_available_;
fml::CFRef<CVPixelBufferRef> last_pixel_buffer_;
sk_sp<SkImage> external_image_;
OSType pixel_format_ = 0;
// |Texture|
void Paint(SkCanvas& canvas,
@ -55,6 +56,10 @@ class IOSExternalTextureMetal final : public Texture {
sk_sp<SkImage> WrapExternalPixelBuffer(fml::CFRef<CVPixelBufferRef> pixel_buffer,
GrDirectContext* context) const;
sk_sp<SkImage> WrapRGBAExternalPixelBuffer(fml::CFRef<CVPixelBufferRef> pixel_buffer,
GrDirectContext* context) const;
sk_sp<SkImage> WrapNV12ExternalPixelBuffer(fml::CFRef<CVPixelBufferRef> pixel_buffer,
GrDirectContext* context) const;
FML_DISALLOW_COPY_AND_ASSIGN(IOSExternalTextureMetal);
};

162
engine/src/flutter/shell/platform/darwin/ios/ios_external_texture_metal.mm
View File

@ -5,6 +5,7 @@
#include "flutter/shell/platform/darwin/ios/ios_external_texture_metal.h"
#include "flutter/fml/logging.h"
#include "third_party/skia/include/core/SkYUVAIndex.h"
#include "third_party/skia/include/gpu/GrBackendSurface.h"
#include "third_party/skia/include/gpu/GrDirectContext.h"
#include "third_party/skia/include/gpu/mtl/GrMtlTypes.h"
@ -35,6 +36,8 @@ void IOSExternalTextureMetal::Paint(SkCanvas& canvas,
auto pixel_buffer = fml::CFRef<CVPixelBufferRef>([external_texture_ copyPixelBuffer]);
if (!pixel_buffer) {
pixel_buffer = std::move(last_pixel_buffer_);
} else {
pixel_format_ = CVPixelBufferGetPixelFormatType(pixel_buffer);
}
// If the application told us there was a texture frame available but did not provide one when
@ -65,21 +68,130 @@ sk_sp<SkImage> IOSExternalTextureMetal::WrapExternalPixelBuffer(
return nullptr;
}
sk_sp<SkImage> image = nullptr;
if (pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange ||
pixel_format_ == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
image = WrapNV12ExternalPixelBuffer(pixel_buffer, context);
} else {
image = WrapRGBAExternalPixelBuffer(pixel_buffer, context);
}
if (!image) {
FML_DLOG(ERROR) << "Could not wrap Metal texture as a Skia image.";
}
return image;
}
sk_sp<SkImage> IOSExternalTextureMetal::WrapNV12ExternalPixelBuffer(
fml::CFRef<CVPixelBufferRef> pixel_buffer,
GrDirectContext* context) const {
auto texture_size =
SkISize::Make(CVPixelBufferGetWidth(pixel_buffer), CVPixelBufferGetHeight(pixel_buffer));
CVMetalTextureRef y_metal_texture_raw = nullptr;
{
auto cv_return =
CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault,
/*textureCache=*/texture_cache_,
/*sourceImage=*/pixel_buffer,
/*textureAttributes=*/nullptr,
/*pixelFormat=*/MTLPixelFormatR8Unorm,
/*width=*/texture_size.width(),
/*height=*/texture_size.height(),
/*planeIndex=*/0u,
/*texture=*/&y_metal_texture_raw);
CVMetalTextureRef metal_texture_raw = NULL;
if (cv_return != kCVReturnSuccess) {
FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cv_return;
return nullptr;
}
}
CVMetalTextureRef uv_metal_texture_raw = nullptr;
{
auto cv_return =
CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault,
/*textureCache=*/texture_cache_,
/*sourceImage=*/pixel_buffer,
/*textureAttributes=*/nullptr,
/*pixelFormat=*/MTLPixelFormatRG8Unorm,
/*width=*/texture_size.width() / 2,
/*height=*/texture_size.height() / 2,
/*planeIndex=*/1u,
/*texture=*/&uv_metal_texture_raw);
if (cv_return != kCVReturnSuccess) {
FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cv_return;
return nullptr;
}
}
fml::CFRef<CVMetalTextureRef> y_metal_texture(y_metal_texture_raw);
GrMtlTextureInfo y_skia_texture_info;
y_skia_texture_info.fTexture = sk_cf_obj<const void*>{
[reinterpret_cast<NSObject*>(CVMetalTextureGetTexture(y_metal_texture)) retain]};
GrBackendTexture y_skia_backend_texture(/*width=*/texture_size.width(),
/*height=*/texture_size.height(),
/*mipMapped=*/GrMipMapped ::kNo,
/*textureInfo=*/y_skia_texture_info);
fml::CFRef<CVMetalTextureRef> uv_metal_texture(uv_metal_texture_raw);
GrMtlTextureInfo uv_skia_texture_info;
uv_skia_texture_info.fTexture = sk_cf_obj<const void*>{
[reinterpret_cast<NSObject*>(CVMetalTextureGetTexture(uv_metal_texture)) retain]};
GrBackendTexture uv_skia_backend_texture(/*width=*/texture_size.width(),
/*height=*/texture_size.height(),
/*mipMapped=*/GrMipMapped ::kNo,
/*textureInfo=*/uv_skia_texture_info);
GrBackendTexture nv12TextureHandles[] = {y_skia_backend_texture, uv_skia_backend_texture};
SkYUVAIndex yuvaIndices[4] = {
SkYUVAIndex{0, SkColorChannel::kR}, // Read Y data from the red channel of the first texture
SkYUVAIndex{1, SkColorChannel::kR}, // Read U data from the red channel of the second texture
SkYUVAIndex{1,
SkColorChannel::kG}, // Read V data from the green channel of the second texture
SkYUVAIndex{-1, SkColorChannel::kA}}; //-1 means to omit the alpha data of YUVA
struct ImageCaptures {
fml::CFRef<CVPixelBufferRef> buffer;
fml::CFRef<CVMetalTextureRef> y_texture;
fml::CFRef<CVMetalTextureRef> uv_texture;
};
auto captures = std::make_unique<ImageCaptures>();
captures->buffer = std::move(pixel_buffer);
captures->y_texture = std::move(y_metal_texture);
captures->uv_texture = std::move(uv_metal_texture);
SkImage::TextureReleaseProc release_proc = [](SkImage::ReleaseContext release_context) {
auto captures = reinterpret_cast<ImageCaptures*>(release_context);
delete captures;
};
sk_sp<SkImage> image = SkImage::MakeFromYUVATextures(
context, kRec601_SkYUVColorSpace, nv12TextureHandles, yuvaIndices, texture_size,
kTopLeft_GrSurfaceOrigin, /*imageColorSpace=*/nullptr, release_proc, captures.release());
return image;
}
sk_sp<SkImage> IOSExternalTextureMetal::WrapRGBAExternalPixelBuffer(
fml::CFRef<CVPixelBufferRef> pixel_buffer,
GrDirectContext* context) const {
auto texture_size =
SkISize::Make(CVPixelBufferGetWidth(pixel_buffer), CVPixelBufferGetHeight(pixel_buffer));
CVMetalTextureRef metal_texture_raw = nullptr;
auto cv_return =
CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, // allocator
texture_cache_, // texture cache
pixel_buffer, // source image
NULL, // texture attributes
MTLPixelFormatBGRA8Unorm, // pixel format
texture_size.width(), // width
texture_size.height(), // height
0u, // plane index
&metal_texture_raw // [out] texture
);
CVMetalTextureCacheCreateTextureFromImage(/*allocator=*/kCFAllocatorDefault,
/*textureCache=*/texture_cache_,
/*sourceImage=*/pixel_buffer,
/*textureAttributes=*/nullptr,
/*pixelFormat=*/MTLPixelFormatBGRA8Unorm,
/*width=*/texture_size.width(),
/*height=*/texture_size.height(),
/*planeIndex=*/0u,
/*texture=*/&metal_texture_raw);
if (cv_return != kCVReturnSuccess) {
FML_DLOG(ERROR) << "Could not create Metal texture from pixel buffer: CVReturn " << cv_return;
@ -92,11 +204,10 @@ sk_sp<SkImage> IOSExternalTextureMetal::WrapExternalPixelBuffer(
skia_texture_info.fTexture = sk_cf_obj<const void*>{
[reinterpret_cast<NSObject*>(CVMetalTextureGetTexture(metal_texture)) retain]};
GrBackendTexture skia_backend_texture(texture_size.width(), // width
texture_size.height(), // height
GrMipMapped ::kNo, // mip-mapped
skia_texture_info // texture info
);
GrBackendTexture skia_backend_texture(/*width=*/texture_size.width(),
/*height=*/texture_size.height(),
/*mipMapped=*/GrMipMapped ::kNo,
/*textureInfo=*/skia_texture_info);
struct ImageCaptures {
fml::CFRef<CVPixelBufferRef> buffer;
@ -112,21 +223,12 @@ sk_sp<SkImage> IOSExternalTextureMetal::WrapExternalPixelBuffer(
delete captures;
};
auto image = SkImage::MakeFromTexture(context, // context
skia_backend_texture, // backend texture
kTopLeft_GrSurfaceOrigin, // origin
kBGRA_8888_SkColorType, // color type
kPremul_SkAlphaType, // alpha type
nullptr, // color space
release_proc, // release proc
captures.release() // release context
);
if (!image) {
FML_DLOG(ERROR) << "Could not wrap Metal texture as a Skia image.";
}
auto image =
SkImage::MakeFromTexture(context, skia_backend_texture, kTopLeft_GrSurfaceOrigin,
kBGRA_8888_SkColorType, kPremul_SkAlphaType,
/*imageColorSpace=*/nullptr, release_proc, captures.release()
);
return image;
}