[Impeller Scene] Render imported meshes (flutter/engine#38097)

* [Impeller Scene] Render imported meshes

* Address comments

engine/src/flutter/impeller/fixtures/BUILD.gn

@@ -33,7 +33,7 @@ impeller_shaders("shader_fixtures") {
   }
 }
 
-scenec("geometry_fixtures") {
+scenec("scene_fixtures") {
   geometry = [ "flutter_logo.glb" ]
   type = "gltf"
 }
@@ -85,8 +85,12 @@ test_fixtures("file_fixtures") {
     fixtures += [ "/System/Library/Fonts/Apple Color Emoji.ttc" ]
   }
   fixtures +=
-      filter_include(get_target_outputs(":runtime_stages"), [ "*.iplr" ])
-  deps = [ ":runtime_stages" ]
+      filter_include(get_target_outputs(":runtime_stages"), [ "*.iplr" ]) +
+      filter_include(get_target_outputs(":scene_fixtures"), [ "*.ipscene" ])
+  deps = [
+    ":runtime_stages",
+    ":scene_fixtures",
+  ]
 }
 
 group("fixtures") {
@@ -94,7 +98,7 @@ group("fixtures") {
 
   public_deps = [
     ":file_fixtures",
-    ":geometry_fixtures",
+    ":scene_fixtures",
     ":shader_fixtures",
   ]
 }

engine/src/flutter/impeller/scene/BUILD.gn

@@ -26,6 +26,7 @@ impeller_component("scene") {
 
   public_deps = [
     "../renderer",
+    "importer:importer_flatbuffers",
     "shaders",
   ]
 
@@ -42,7 +43,5 @@ impeller_component("scene_unittests") {
     "../fixtures",
     "../playground:playground_test",
     "//flutter/testing:testing_lib",
-
-    #"//third_party/tinygltf",
   ]
 }

engine/src/flutter/impeller/scene/geometry.cc

@@ -4,13 +4,19 @@
 
 #include "impeller/scene/geometry.h"
 
+#include <iostream>
 #include <memory>
+#include <ostream>
 
 #include "impeller/geometry/point.h"
 #include "impeller/geometry/vector.h"
+#include "impeller/renderer/device_buffer_descriptor.h"
 #include "impeller/renderer/formats.h"
+#include "impeller/renderer/vertex_buffer.h"
 #include "impeller/renderer/vertex_buffer_builder.h"
+#include "impeller/scene/importer/scene_flatbuffers.h"
 #include "impeller/scene/shaders/geometry.vert.h"
+#include "third_party/flatbuffers/include/flatbuffers/vector.h"
 
 namespace impeller {
 namespace scene {
@@ -19,16 +25,78 @@ namespace scene {
 /// Geometry
 ///
 
+Geometry::~Geometry() = default;
+
 std::shared_ptr<CuboidGeometry> Geometry::MakeCuboid(Vector3 size) {
   auto result = std::make_shared<CuboidGeometry>();
   result->SetSize(size);
   return result;
 }
 
+std::shared_ptr<VertexBufferGeometry> Geometry::MakeVertexBuffer(
+    VertexBuffer vertex_buffer) {
+  auto result = std::make_shared<VertexBufferGeometry>();
+  result->SetVertexBuffer(std::move(vertex_buffer));
+  return result;
+}
+
+std::shared_ptr<VertexBufferGeometry> Geometry::MakeFromFBMesh(
+    const fb::StaticMesh& mesh,
+    Allocator& allocator) {
+  IndexType index_type;
+  switch (mesh.indices()->type()) {
+    case fb::IndicesType::k16Bit:
+      index_type = IndexType::k16bit;
+      break;
+    case fb::IndicesType::k32Bit:
+      index_type = IndexType::k32bit;
+      break;
+  }
+
+  const size_t vertices_bytes = mesh.vertices()->size() * sizeof(fb::Vertex);
+  const size_t indices_bytes = mesh.indices()->data()->size();
+  if (vertices_bytes == 0 || indices_bytes == 0) {
+    return nullptr;
+  }
+
+  DeviceBufferDescriptor buffer_desc;
+  buffer_desc.size = vertices_bytes * indices_bytes;
+  buffer_desc.storage_mode = StorageMode::kHostVisible;
+
+  auto buffer = allocator.CreateBuffer(buffer_desc);
+  buffer->SetLabel("Mesh vertices+indices");
+
+  const uint8_t* vertices_start =
+      reinterpret_cast<const uint8_t*>(mesh.vertices()->Get(0));
+  const uint8_t* indices_start =
+      reinterpret_cast<const uint8_t*>(mesh.indices()->data()->Data());
+
+  if (!buffer->CopyHostBuffer(vertices_start, Range(0, vertices_bytes))) {
+    return nullptr;
+  }
+  if (!buffer->CopyHostBuffer(indices_start, Range(0, indices_bytes),
+                              vertices_bytes)) {
+    return nullptr;
+  }
+
+  VertexBuffer vertex_buffer = {
+      .vertex_buffer = {.buffer = buffer, .range = Range(0, vertices_bytes)},
+      .index_buffer = {.buffer = buffer,
+                       .range = Range(vertices_bytes, indices_bytes)},
+      .index_count = mesh.indices()->count(),
+      .index_type = index_type,
+  };
+  return MakeVertexBuffer(std::move(vertex_buffer));
+}
+
 //------------------------------------------------------------------------------
 /// CuboidGeometry
 ///
 
+CuboidGeometry::CuboidGeometry() = default;
+
+CuboidGeometry::~CuboidGeometry() = default;
+
 void CuboidGeometry::SetSize(Vector3 size) {
   size_ = size;
 }
@@ -38,15 +106,37 @@ VertexBuffer CuboidGeometry::GetVertexBuffer(Allocator& allocator) const {
   // Layout: position, normal, tangent, uv
   builder.AddVertices({
       // Front.
-      {Vector3(0, 0, 0), Vector3(0, 0, -1), Vector3(1, 0, 0), Point(0, 0)},
-      {Vector3(1, 0, 0), Vector3(0, 0, -1), Vector3(1, 0, 0), Point(1, 0)},
-      {Vector3(1, 1, 0), Vector3(0, 0, -1), Vector3(1, 0, 0), Point(1, 1)},
-      {Vector3(1, 1, 0), Vector3(0, 0, -1), Vector3(1, 0, 0), Point(1, 1)},
-      {Vector3(0, 1, 0), Vector3(0, 0, -1), Vector3(1, 0, 0), Point(0, 1)},
-      {Vector3(0, 0, 0), Vector3(0, 0, -1), Vector3(1, 0, 0), Point(0, 0)},
+      {Vector3(0, 0, 0), Vector3(0, 0, -1), Vector3(1, 0, 0), Point(0, 0),
+       Color::White()},
+      {Vector3(1, 0, 0), Vector3(0, 0, -1), Vector3(1, 0, 0), Point(1, 0),
+       Color::White()},
+      {Vector3(1, 1, 0), Vector3(0, 0, -1), Vector3(1, 0, 0), Point(1, 1),
+       Color::White()},
+      {Vector3(1, 1, 0), Vector3(0, 0, -1), Vector3(1, 0, 0), Point(1, 1),
+       Color::White()},
+      {Vector3(0, 1, 0), Vector3(0, 0, -1), Vector3(1, 0, 0), Point(0, 1),
+       Color::White()},
+      {Vector3(0, 0, 0), Vector3(0, 0, -1), Vector3(1, 0, 0), Point(0, 0),
+       Color::White()},
   });
   return builder.CreateVertexBuffer(allocator);
 }
 
+//------------------------------------------------------------------------------
+/// VertexBufferGeometry
+///
+
+VertexBufferGeometry::VertexBufferGeometry() = default;
+
+VertexBufferGeometry::~VertexBufferGeometry() = default;
+
+void VertexBufferGeometry::SetVertexBuffer(VertexBuffer vertex_buffer) {
+  vertex_buffer_ = std::move(vertex_buffer);
+}
+
+VertexBuffer VertexBufferGeometry::GetVertexBuffer(Allocator& allocator) const {
+  return vertex_buffer_;
+}
+
 }  // namespace scene
 }  // namespace impeller

engine/src/flutter/impeller/scene/geometry.h

@@ -6,30 +6,67 @@
 
 #include <memory>
 
+#include "flutter/fml/macros.h"
 #include "impeller/geometry/vector.h"
 #include "impeller/renderer/allocator.h"
+#include "impeller/renderer/device_buffer.h"
 #include "impeller/renderer/vertex_buffer.h"
+#include "impeller/scene/importer/scene_flatbuffers.h"
 
 namespace impeller {
 namespace scene {
 
 class CuboidGeometry;
+class VertexBufferGeometry;
 
 class Geometry {
  public:
+  virtual ~Geometry();
+
   static std::shared_ptr<CuboidGeometry> MakeCuboid(Vector3 size);
 
+  static std::shared_ptr<VertexBufferGeometry> MakeVertexBuffer(
+      VertexBuffer vertex_buffer);
+
+  static std::shared_ptr<VertexBufferGeometry> MakeFromFBMesh(
+      const fb::StaticMesh& mesh,
+      Allocator& allocator);
+
   virtual VertexBuffer GetVertexBuffer(Allocator& allocator) const = 0;
 };
 
 class CuboidGeometry final : public Geometry {
  public:
+  CuboidGeometry();
+
+  ~CuboidGeometry() override;
+
   void SetSize(Vector3 size);
 
+  // |Geometry|
   VertexBuffer GetVertexBuffer(Allocator& allocator) const override;
 
  private:
   Vector3 size_;
+
+  FML_DISALLOW_COPY_AND_ASSIGN(CuboidGeometry);
+};
+
+class VertexBufferGeometry final : public Geometry {
+ public:
+  VertexBufferGeometry();
+
+  ~VertexBufferGeometry() override;
+
+  void SetVertexBuffer(VertexBuffer vertex_buffer);
+
+  // |Geometry|
+  VertexBuffer GetVertexBuffer(Allocator& allocator) const override;
+
+ private:
+  VertexBuffer vertex_buffer_;
+
+  FML_DISALLOW_COPY_AND_ASSIGN(VertexBufferGeometry);
 };
 
 }  // namespace scene

engine/src/flutter/impeller/scene/importer/importer_gltf.cc

@@ -108,10 +108,28 @@ static bool ProcessStaticMesh(const tinygltf::Model& gltf,
 
   auto index_accessor = gltf.accessors[primitive.indices];
   auto index_view = gltf.bufferViews[index_accessor.bufferView];
-  static_mesh.indices.resize(index_accessor.count);
+
+  auto indices = std::make_unique<fb::IndicesT>();
+
+  switch (index_accessor.componentType) {
+    case TINYGLTF_COMPONENT_TYPE_UNSIGNED_SHORT:
+      indices->type = fb::IndicesType::k16Bit;
+      break;
+    case TINYGLTF_COMPONENT_TYPE_UNSIGNED_INT:
+      indices->type = fb::IndicesType::k32Bit;
+      break;
+    default:
+      std::cerr << "Mesh primitive has unsupported index type "
+                << index_accessor.componentType << ". Skipping.";
+      return false;
+  }
+  indices->count = index_accessor.count;
+  indices->data.resize(index_view.byteLength);
   const auto* index_buffer =
       &gltf.buffers[index_view.buffer].data[index_view.byteOffset];
-  std::memcpy(static_mesh.indices.data(), index_buffer, index_view.byteLength);
+  std::memcpy(indices->data.data(), index_buffer, indices->data.size());
+
+  static_mesh.indices = std::move(indices);
 
   return true;
 }

engine/src/flutter/impeller/scene/importer/importer_unittests.cc

@@ -28,7 +28,11 @@ TEST(ImporterTest, CanParseGLTF) {
 
   ASSERT_EQ(node.meshes.size(), 1u);
   auto& mesh = *node.meshes[0];
-  ASSERT_EQ(mesh.indices.size(), 918u);
+  ASSERT_EQ(mesh.indices->count, 918u);
+
+  uint16_t first_index =
+      *reinterpret_cast<uint16_t*>(mesh.indices->data.data());
+  ASSERT_EQ(first_index, 45u);
 
   ASSERT_EQ(mesh.vertices.size(), 260u);
   auto& vertex = mesh.vertices[0];

engine/src/flutter/impeller/scene/importer/scene.fbs

@@ -33,6 +33,8 @@ struct Matrix {
   m: [float:16];
 }
 
+// This attribute layout is expected to be identical to that within
+// `impeller/scene/shaders/geometry.vert`.
 struct Vertex {
   position: Vec3;
   normal: Vec3;
@@ -41,6 +43,17 @@ struct Vertex {
   color: Color;
 }
 
+enum IndicesType:byte {
+  k16Bit,
+  k32Bit,
+}
+
+table Indices {
+  data: [ubyte];
+  count: uint32;
+  type: IndicesType;
+}
+
 table Texture {
   // TODO(bdero): Allow optional image data embedding.
   uri: string;
@@ -54,7 +67,7 @@ table Material {
 
 table StaticMesh {
   vertices: [Vertex];
-  indices: [uint32];
+  indices: Indices;
   material: Material;
 }
 

engine/src/flutter/impeller/scene/importer/scenec_main.cc

@@ -13,6 +13,7 @@
 #include "impeller/base/strings.h"
 #include "impeller/compiler/utilities.h"
 #include "impeller/scene/importer/importer.h"
+#include "impeller/scene/importer/scene_flatbuffers.h"
 #include "impeller/scene/importer/switches.h"
 #include "impeller/scene/importer/types.h"
 
@@ -74,7 +75,7 @@ bool Main(const fml::CommandLine& command_line) {
   }
 
   flatbuffers::FlatBufferBuilder builder;
-  builder.Finish(fb::Scene::Pack(builder, &scene));
+  builder.Finish(fb::Scene::Pack(builder, &scene), fb::SceneIdentifier());
 
   auto output_file_name = std::filesystem::absolute(
       std::filesystem::current_path() / switches.output_file_name);

engine/src/flutter/impeller/scene/scene_unittests.cc

@@ -9,12 +9,13 @@
 #include "impeller/geometry/vector.h"
 #include "impeller/playground/playground.h"
 #include "impeller/playground/playground_test.h"
-
 #include "impeller/scene/camera.h"
 #include "impeller/scene/geometry.h"
+#include "impeller/scene/importer/scene_flatbuffers.h"
 #include "impeller/scene/material.h"
 #include "impeller/scene/scene.h"
 #include "impeller/scene/static_mesh_entity.h"
+#include "third_party/flatbuffers/include/flatbuffers/verifier.h"
 
 // #include "third_party/tinygltf/tiny_gltf.h"
 
@@ -60,6 +61,48 @@ TEST_P(SceneTest, CuboidUnlit) {
   OpenPlaygroundHere(callback);
 }
 
+TEST_P(SceneTest, GLTFScene) {
+  auto allocator = GetContext()->GetResourceAllocator();
+
+  auto mapping =
+      flutter::testing::OpenFixtureAsMapping("flutter_logo.glb.ipscene");
+
+  flatbuffers::Verifier verifier(mapping->GetMapping(), mapping->GetSize());
+  ASSERT_TRUE(fb::VerifySceneBuffer(verifier));
+
+  // TODO(bdero): Add full scene deserialization utilities.
+  const auto* fb_scene = fb::GetScene(mapping->GetMapping());
+  const auto fb_nodes = fb_scene->children();
+  ASSERT_EQ(fb_nodes->size(), 1u);
+  const auto fb_meshes = fb_nodes->begin()->meshes();
+  ASSERT_EQ(fb_meshes->size(), 1u);
+  const auto* fb_mesh = fb_meshes->Get(0);
+  auto geometry = Geometry::MakeFromFBMesh(*fb_mesh, *allocator);
+  ASSERT_NE(geometry, nullptr);
+
+  Renderer::RenderCallback callback = [&](RenderTarget& render_target) {
+    auto scene = Scene(GetContext());
+
+    auto mesh = SceneEntity::MakeStaticMesh();
+    mesh->SetMaterial(Material::MakeUnlit());
+    mesh->SetGeometry(geometry);
+    scene.Add(mesh);
+
+    // Face towards the +Z direction (+X right, +Y up).
+    auto camera = Camera::MakePerspective(
+                      /* fov */ Radians(kPiOver4),
+                      /* position */ {2, 2, -5})
+                      .LookAt(
+                          /* target */ Vector3(),
+                          /* up */ {0, 1, 0});
+
+    scene.Render(render_target, camera);
+    return true;
+  };
+
+  OpenPlaygroundHere(callback);
+}
+
 }  // namespace testing
 }  // namespace scene
 }  // namespace impeller

engine/src/flutter/impeller/scene/shaders/geometry.vert

@@ -7,20 +7,26 @@ uniform VertInfo {
 }
 vert_info;
 
+// This attribute layout is expected to be identical to that within
+// `impeller/scene/importer/scene.fbs`.
 in vec3 position;
 in vec3 normal;
-in vec3 tangent;
+in vec4 tangent;
 in vec2 texture_coords;
+in vec4 color;
 
 out vec3 v_position;
 out mat3 v_tangent_space;
 out vec2 v_texture_coords;
+out vec4 v_color;
 
 void main() {
   gl_Position = vert_info.mvp * vec4(position, 1.0);
   v_position = gl_Position.xyz;
 
+  vec3 lh_tangent = tangent.xyz * tangent.w;
   v_tangent_space =
-      mat3(vert_info.mvp) * mat3(tangent, cross(normal, tangent), normal);
+      mat3(vert_info.mvp) * mat3(lh_tangent, cross(normal, lh_tangent), normal);
   v_texture_coords = texture_coords;
+  v_color = color;
 }

engine/src/flutter/impeller/scene/shaders/unlit.frag

@@ -12,9 +12,11 @@ uniform sampler2D base_color_texture;
 in vec3 v_position;
 in mat3 v_tangent_space;
 in vec2 v_texture_coords;
+in vec4 v_color;
 
 out vec4 frag_color;
 
 void main() {
-  frag_color = texture(base_color_texture, v_texture_coords) * frag_info.color;
+  frag_color =
+      texture(base_color_texture, v_texture_coords) * v_color * frag_info.color;
 }

engine/src/flutter/impeller/tools/impeller.gni

@@ -656,7 +656,7 @@ template("scenec") {
       "--input-type=$input_type",
     ]
 
-    output = "$generated_dir/{{source_file_part}}.ipmesh"
+    output = "$generated_dir/{{source_file_part}}.ipscene"
     output_path = rebase_path(output, root_build_dir)
 
     args += [ "--output=$output_path" ]