diff --git a/engine/src/flutter/ci/licenses_golden/licenses_flutter b/engine/src/flutter/ci/licenses_golden/licenses_flutter index 15e97c6b8f7..1d5cee94210 100755 --- a/engine/src/flutter/ci/licenses_golden/licenses_flutter +++ b/engine/src/flutter/ci/licenses_golden/licenses_flutter @@ -162,6 +162,9 @@ FILE: ../../../flutter/fml/file.h FILE: ../../../flutter/fml/file_unittest.cc FILE: ../../../flutter/fml/hash_combine.h FILE: ../../../flutter/fml/hash_combine_unittests.cc +FILE: ../../../flutter/fml/hex_codec.cc +FILE: ../../../flutter/fml/hex_codec.h +FILE: ../../../flutter/fml/hex_codec_unittest.cc FILE: ../../../flutter/fml/icu_util.cc FILE: ../../../flutter/fml/icu_util.h FILE: ../../../flutter/fml/log_level.h diff --git a/engine/src/flutter/common/graphics/BUILD.gn b/engine/src/flutter/common/graphics/BUILD.gn index 5afc6cae533..6ac06054f22 100644 --- a/engine/src/flutter/common/graphics/BUILD.gn +++ b/engine/src/flutter/common/graphics/BUILD.gn @@ -22,6 +22,7 @@ source_set("graphics") { "//flutter/assets", "//flutter/fml", "//flutter/shell/version:version", + "//third_party/boringssl", "//third_party/rapidjson", "//third_party/skia", ] diff --git a/engine/src/flutter/common/graphics/persistent_cache.cc b/engine/src/flutter/common/graphics/persistent_cache.cc index 69c906fdd3f..88889b757b7 100644 --- a/engine/src/flutter/common/graphics/persistent_cache.cc +++ b/engine/src/flutter/common/graphics/persistent_cache.cc @@ -11,12 +11,14 @@ #include "flutter/fml/base32.h" #include "flutter/fml/file.h" +#include "flutter/fml/hex_codec.h" #include "flutter/fml/logging.h" #include "flutter/fml/make_copyable.h" #include "flutter/fml/mapping.h" #include "flutter/fml/paths.h" #include "flutter/fml/trace_event.h" #include "flutter/shell/version/version.h" +#include "openssl/sha.h" #include "rapidjson/document.h" #include "third_party/skia/include/gpu/GrDirectContext.h" #include "third_party/skia/include/utils/SkBase64.h" @@ -30,21 +32,17 @@ std::shared_ptr PersistentCache::asset_manager_; std::mutex PersistentCache::instance_mutex_; std::unique_ptr PersistentCache::gPersistentCache; -std::string PersistentCache::SkKeyToFilePath(const SkData& data) { - if (data.data() == nullptr || data.size() == 0) { +std::string PersistentCache::SkKeyToFilePath(const SkData& key) { + if (key.data() == nullptr || key.size() == 0) { return ""; } - std::string_view view(reinterpret_cast(data.data()), - data.size()); + uint8_t sha_digest[SHA_DIGEST_LENGTH]; + SHA1(static_cast(key.data()), key.size(), sha_digest); - auto encode_result = fml::Base32Encode(view); - - if (!encode_result.first) { - return ""; - } - - return encode_result.second; + std::string_view view(reinterpret_cast(sha_digest), + SHA_DIGEST_LENGTH); + return fml::HexEncode(view); } bool PersistentCache::gIsReadOnly = false; @@ -205,7 +203,7 @@ size_t PersistentCache::PrecompileKnownSkSLs(GrDirectContext* context) const { size_t precompiled_count = 0; for (const auto& sksl : known_sksls) { TRACE_EVENT0("flutter", "PrecompilingSkSL"); - if (context->precompileShader(*sksl.first, *sksl.second)) { + if (context->precompileShader(*sksl.key, *sksl.value)) { precompiled_count++; } } @@ -221,10 +219,9 @@ std::vector PersistentCache::LoadSkSLs() const { std::vector result; fml::FileVisitor visitor = [&result](const fml::UniqueFD& directory, const std::string& filename) { - sk_sp key = ParseBase32(filename); - sk_sp data = LoadFile(directory, filename); - if (key != nullptr && data != nullptr) { - result.push_back({key, data}); + SkSLCache cache = LoadFile(directory, filename, true); + if (cache.key != nullptr && cache.value != nullptr) { + result.push_back(cache); } else { FML_LOG(ERROR) << "Failed to load: " << filename; } @@ -291,17 +288,38 @@ bool PersistentCache::IsValid() const { return cache_directory_ && cache_directory_->is_valid(); } -sk_sp PersistentCache::LoadFile(const fml::UniqueFD& dir, - const std::string& file_name) { +PersistentCache::SkSLCache PersistentCache::LoadFile( + const fml::UniqueFD& dir, + const std::string& file_name, + bool need_key) { + SkSLCache result; auto file = fml::OpenFileReadOnly(dir, file_name.c_str()); if (!file.is_valid()) { - return nullptr; + return result; } auto mapping = std::make_unique(file); - if (mapping->GetSize() == 0) { - return nullptr; + if (mapping->GetSize() < sizeof(CacheObjectHeader)) { + return result; } - return SkData::MakeWithCopy(mapping->GetMapping(), mapping->GetSize()); + const CacheObjectHeader* header = + reinterpret_cast(mapping->GetMapping()); + if (header->signature != CacheObjectHeader::kSignature || + header->version != CacheObjectHeader::kVersion1) { + FML_LOG(INFO) << "Persistent cache header is corrupt: " << file_name; + return result; + } + if (mapping->GetSize() < sizeof(CacheObjectHeader) + header->key_size) { + FML_LOG(INFO) << "Persistent cache size is corrupt: " << file_name; + return result; + } + if (need_key) { + result.key = SkData::MakeWithCopy( + mapping->GetMapping() + sizeof(CacheObjectHeader), header->key_size); + } + size_t value_offset = sizeof(CacheObjectHeader) + header->key_size; + result.value = SkData::MakeWithCopy(mapping->GetMapping() + value_offset, + mapping->GetSize() - value_offset); + return result; } // |GrContextOptions::PersistentCache| @@ -314,7 +332,8 @@ sk_sp PersistentCache::load(const SkData& key) { if (file_name.size() == 0) { return nullptr; } - auto result = PersistentCache::LoadFile(*cache_directory_, file_name); + auto result = + PersistentCache::LoadFile(*cache_directory_, file_name, false).value; if (result != nullptr) { TRACE_EVENT0("flutter", "PersistentCacheLoadHit"); } @@ -349,6 +368,26 @@ static void PersistentCacheStore(fml::RefPtr worker, } } +std::unique_ptr PersistentCache::BuildCacheObject( + const SkData& key, + const SkData& data) { + size_t total_size = sizeof(CacheObjectHeader) + key.size() + data.size(); + uint8_t* mapping_buf = reinterpret_cast(malloc(total_size)); + if (!mapping_buf) { + return nullptr; + } + auto mapping = std::make_unique(mapping_buf, total_size); + + CacheObjectHeader header(key.size()); + memcpy(mapping_buf, &header, sizeof(CacheObjectHeader)); + mapping_buf += sizeof(CacheObjectHeader); + memcpy(mapping_buf, key.data(), key.size()); + mapping_buf += key.size(); + memcpy(mapping_buf, data.data(), data.size()); + + return mapping; +} + // |GrContextOptions::PersistentCache| void PersistentCache::store(const SkData& key, const SkData& data) { stored_new_shaders_ = true; @@ -367,10 +406,8 @@ void PersistentCache::store(const SkData& key, const SkData& data) { return; } - auto mapping = std::make_unique( - std::vector{data.bytes(), data.bytes() + data.size()}); - - if (mapping == nullptr || mapping->GetSize() == 0) { + std::unique_ptr mapping = BuildCacheObject(key, data); + if (!mapping) { return; } diff --git a/engine/src/flutter/common/graphics/persistent_cache.h b/engine/src/flutter/common/graphics/persistent_cache.h index 00b6c299b58..b10651876b9 100644 --- a/engine/src/flutter/common/graphics/persistent_cache.h +++ b/engine/src/flutter/common/graphics/persistent_cache.h @@ -45,7 +45,26 @@ class PersistentCache : public GrContextOptions::PersistentCache { // // This is used to specify persistent cache filenames and service protocol // json keys. - static std::string SkKeyToFilePath(const SkData& data); + static std::string SkKeyToFilePath(const SkData& key); + + // Allocate a MallocMapping containing the given key and value in the file + // format used by the cache. + static std::unique_ptr BuildCacheObject( + const SkData& key, + const SkData& data); + + // Header written into the files used to store cached Skia objects. + struct CacheObjectHeader { + // A prefix used to identify the cache object file format. + static const uint32_t kSignature = 0xA869593F; + static const uint32_t kVersion1 = 1; + + CacheObjectHeader(uint32_t p_key_size) : key_size(p_key_size) {} + + uint32_t signature = kSignature; + uint32_t version = kVersion1; + uint32_t key_size; + }; ~PersistentCache() override; @@ -69,7 +88,10 @@ class PersistentCache : public GrContextOptions::PersistentCache { // |GrContextOptions::PersistentCache| sk_sp load(const SkData& key) override; - using SkSLCache = std::pair, sk_sp>; + struct SkSLCache { + sk_sp key; + sk_sp value; + }; /// Load all the SkSL shader caches in the right directory. std::vector LoadSkSLs() const; @@ -135,8 +157,9 @@ class PersistentCache : public GrContextOptions::PersistentCache { bool stored_new_shaders_ = false; bool is_dumping_skp_ = false; - static sk_sp LoadFile(const fml::UniqueFD& dir, - const std::string& filen_ame); + static SkSLCache LoadFile(const fml::UniqueFD& dir, + const std::string& file_name, + bool need_key); bool IsValid() const; diff --git a/engine/src/flutter/fml/BUILD.gn b/engine/src/flutter/fml/BUILD.gn index 5c8432e827b..a8cb5ae8c80 100644 --- a/engine/src/flutter/fml/BUILD.gn +++ b/engine/src/flutter/fml/BUILD.gn @@ -24,6 +24,8 @@ source_set("fml") { "file.cc", "file.h", "hash_combine.h", + "hex_codec.cc", + "hex_codec.h", "icu_util.cc", "icu_util.h", "log_level.h", @@ -269,6 +271,7 @@ if (enable_unittests) { "command_line_unittest.cc", "file_unittest.cc", "hash_combine_unittests.cc", + "hex_codec_unittest.cc", "logging_unittests.cc", "mapping_unittests.cc", "memory/ref_counted_unittest.cc", diff --git a/engine/src/flutter/fml/hex_codec.cc b/engine/src/flutter/fml/hex_codec.cc new file mode 100644 index 00000000000..7c5fb0f27bc --- /dev/null +++ b/engine/src/flutter/fml/hex_codec.cc @@ -0,0 +1,24 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/fml/base32.h" + +#include + +namespace fml { + +static constexpr char kEncoding[] = "0123456789abcdef"; + +std::string HexEncode(std::string_view input) { + std::string result; + result.reserve(input.size() * 2); + for (char c : input) { + uint8_t b = static_cast(c); + result.push_back(kEncoding[b >> 4]); + result.push_back(kEncoding[b & 0xF]); + } + return result; +} + +} // namespace fml diff --git a/engine/src/flutter/fml/hex_codec.h b/engine/src/flutter/fml/hex_codec.h new file mode 100644 index 00000000000..4de83fb154b --- /dev/null +++ b/engine/src/flutter/fml/hex_codec.h @@ -0,0 +1,16 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_FML_HEX_CODEC_H_ +#define FLUTTER_FML_HEX_CODEC_H_ + +#include + +namespace fml { + +std::string HexEncode(std::string_view input); + +} // namespace fml + +#endif // FLUTTER_FML_HEX_CODEC_H_ diff --git a/engine/src/flutter/fml/hex_codec_unittest.cc b/engine/src/flutter/fml/hex_codec_unittest.cc new file mode 100644 index 00000000000..05c65f8e079 --- /dev/null +++ b/engine/src/flutter/fml/hex_codec_unittest.cc @@ -0,0 +1,31 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "flutter/fml/hex_codec.h" + +#include + +#include "gtest/gtest.h" + +TEST(HexCodecTest, CanEncode) { + { + auto result = fml::HexEncode("hello"); + ASSERT_EQ(result, "68656c6c6f"); + } + + { + auto result = fml::HexEncode(""); + ASSERT_EQ(result, ""); + } + + { + auto result = fml::HexEncode("1"); + ASSERT_EQ(result, "31"); + } + + { + auto result = fml::HexEncode(std::string_view("\xFF\xFE\x00\x01", 4)); + ASSERT_EQ(result, "fffe0001"); + } +} diff --git a/engine/src/flutter/shell/common/persistent_cache_unittests.cc b/engine/src/flutter/shell/common/persistent_cache_unittests.cc index a83f42649f1..9a1daf2a71c 100644 --- a/engine/src/flutter/shell/common/persistent_cache_unittests.cc +++ b/engine/src/flutter/shell/common/persistent_cache_unittests.cc @@ -253,14 +253,14 @@ TEST_F(PersistentCacheTest, CanLoadSkSLsFromAsset) { // Make sure that the 2 shaders are sorted by their keys. Their keys should // be "A" and "B" (decoded from "II" and "IE"). - if (shaders[0].first->bytes()[0] == 'B') { + if (shaders[0].key->bytes()[0] == 'B') { std::swap(shaders[0], shaders[1]); } - CheckTextSkData(shaders[0].first, "A"); - CheckTextSkData(shaders[1].first, "B"); - CheckTextSkData(shaders[0].second, "x"); - CheckTextSkData(shaders[1].second, "y"); + CheckTextSkData(shaders[0].key, "A"); + CheckTextSkData(shaders[1].key, "B"); + CheckTextSkData(shaders[0].value, "x"); + CheckTextSkData(shaders[1].value, "y"); } // Cleanup. diff --git a/engine/src/flutter/shell/common/shell.cc b/engine/src/flutter/shell/common/shell.cc index 8d1b52608e5..376524710d7 100644 --- a/engine/src/flutter/shell/common/shell.cc +++ b/engine/src/flutter/shell/common/shell.cc @@ -11,6 +11,7 @@ #include "flutter/assets/directory_asset_bundle.h" #include "flutter/common/graphics/persistent_cache.h" +#include "flutter/fml/base32.h" #include "flutter/fml/file.h" #include "flutter/fml/icu_util.h" #include "flutter/fml/log_settings.h" @@ -1679,14 +1680,19 @@ bool Shell::OnServiceProtocolGetSkSLs( std::vector sksls = persistent_cache->LoadSkSLs(); for (const auto& sksl : sksls) { size_t b64_size = - SkBase64::Encode(sksl.second->data(), sksl.second->size(), nullptr); + SkBase64::Encode(sksl.value->data(), sksl.value->size(), nullptr); sk_sp b64_data = SkData::MakeUninitialized(b64_size + 1); char* b64_char = static_cast(b64_data->writable_data()); - SkBase64::Encode(sksl.second->data(), sksl.second->size(), b64_char); + SkBase64::Encode(sksl.value->data(), sksl.value->size(), b64_char); b64_char[b64_size] = 0; // make it null terminated for printing rapidjson::Value shader_value(b64_char, response->GetAllocator()); - rapidjson::Value shader_key(PersistentCache::SkKeyToFilePath(*sksl.first), - response->GetAllocator()); + std::string_view key_view(reinterpret_cast(sksl.key->data()), + sksl.key->size()); + auto encode_result = fml::Base32Encode(key_view); + if (!encode_result.first) { + continue; + } + rapidjson::Value shader_key(encode_result.second, response->GetAllocator()); shaders_json.AddMember(shader_key, shader_value, response->GetAllocator()); } response->AddMember("SkSLs", shaders_json, response->GetAllocator()); diff --git a/engine/src/flutter/shell/common/shell_unittests.cc b/engine/src/flutter/shell/common/shell_unittests.cc index eff95541eb1..fb3e58a1b8b 100644 --- a/engine/src/flutter/shell/common/shell_unittests.cc +++ b/engine/src/flutter/shell/common/shell_unittests.cc @@ -12,6 +12,7 @@ #include #include "assets/directory_asset_bundle.h" +#include "common/graphics/persistent_cache.h" #include "flutter/common/graphics/persistent_cache.h" #include "flutter/flow/layers/layer_tree.h" #include "flutter/flow/layers/picture_layer.h" @@ -2012,14 +2013,24 @@ TEST_F(ShellTest, OnServiceProtocolGetSkSLsWorks) { PersistentCache::kSkSLSubdirName}; auto sksl_dir = fml::CreateDirectory(base_dir.fd(), components, fml::FilePermission::kReadWrite); - const std::string x = "x"; - const std::string y = "y"; - auto x_data = std::make_unique( - std::vector{x.begin(), x.end()}); - auto y_data = std::make_unique( - std::vector{y.begin(), y.end()}); - ASSERT_TRUE(fml::WriteAtomically(sksl_dir, "IE", *x_data)); - ASSERT_TRUE(fml::WriteAtomically(sksl_dir, "II", *y_data)); + const std::string x_key_str = "A"; + const std::string x_value_str = "x"; + sk_sp x_key = + SkData::MakeWithCopy(x_key_str.data(), x_key_str.size()); + sk_sp x_value = + SkData::MakeWithCopy(x_value_str.data(), x_value_str.size()); + auto x_data = PersistentCache::BuildCacheObject(*x_key, *x_value); + + const std::string y_key_str = "B"; + const std::string y_value_str = "y"; + sk_sp y_key = + SkData::MakeWithCopy(y_key_str.data(), y_key_str.size()); + sk_sp y_value = + SkData::MakeWithCopy(y_value_str.data(), y_value_str.size()); + auto y_data = PersistentCache::BuildCacheObject(*y_key, *y_value); + + ASSERT_TRUE(fml::WriteAtomically(sksl_dir, "x_cache", *x_data)); + ASSERT_TRUE(fml::WriteAtomically(sksl_dir, "y_cache", *y_data)); Settings settings = CreateSettingsForFixture(); std::unique_ptr shell = CreateShell(settings);