diff --git a/engine/src/flutter/app/HyphTool.cpp b/engine/src/flutter/app/HyphTool.cpp index cdb1466f409..403d37439a8 100644 --- a/engine/src/flutter/app/HyphTool.cpp +++ b/engine/src/flutter/app/HyphTool.cpp @@ -11,7 +11,7 @@ using minikin::HyphenationType; using minikin::Hyphenator; -Hyphenator* loadHybFile(const char* fn) { +Hyphenator* loadHybFile(const char* fn, int minPrefix, int minSuffix) { struct stat statbuf; int status = stat(fn, &statbuf); if (status < 0) { @@ -32,11 +32,11 @@ Hyphenator* loadHybFile(const char* fn) { delete[] buf; return nullptr; } - return Hyphenator::loadBinary(buf); + return Hyphenator::loadBinary(buf, minPrefix, minSuffix); } int main(int argc, char** argv) { - Hyphenator* hyph = loadHybFile("/tmp/en.hyb"); // should also be configurable + Hyphenator* hyph = loadHybFile("/tmp/en.hyb", 2, 3); // should also be configurable std::vector result; std::vector word; if (argc < 2) { diff --git a/engine/src/flutter/include/minikin/Hyphenator.h b/engine/src/flutter/include/minikin/Hyphenator.h index ea81813745a..2b8ccb71a7d 100644 --- a/engine/src/flutter/include/minikin/Hyphenator.h +++ b/engine/src/flutter/include/minikin/Hyphenator.h @@ -136,7 +136,7 @@ public: // at least as long as the Hyphenator object. // Note: nullptr is valid input, in which case the hyphenator only processes soft hyphens. - static Hyphenator* loadBinary(const uint8_t* patternData); + static Hyphenator* loadBinary(const uint8_t* patternData, size_t minPrefix, size_t minSuffix); private: // apply various hyphenation rules including hard and soft hyphens, ignoring patterns @@ -153,17 +153,13 @@ private: void hyphenateFromCodes(HyphenationType* result, const uint16_t* codes, size_t len, HyphenationType hyphenValue); - // TODO: these should become parameters, as they might vary by locale, screen size, and - // possibly explicit user control. - static const int MIN_PREFIX = 2; - static const int MIN_SUFFIX = 3; - // See also LONGEST_HYPHENATED_WORD in LineBreaker.cpp. Here the constant is used so // that temporary buffers can be stack-allocated without waste, which is a slightly // different use case. It measures UTF-16 code units. static const size_t MAX_HYPHENATED_SIZE = 64; const uint8_t* patternData; + size_t minPrefix, minSuffix; // accessors for binary data const Header* getHeader() const { diff --git a/engine/src/flutter/libs/minikin/Hyphenator.cpp b/engine/src/flutter/libs/minikin/Hyphenator.cpp index 5ec82feb506..0605b278312 100644 --- a/engine/src/flutter/libs/minikin/Hyphenator.cpp +++ b/engine/src/flutter/libs/minikin/Hyphenator.cpp @@ -108,9 +108,11 @@ struct Header { } }; -Hyphenator* Hyphenator::loadBinary(const uint8_t* patternData) { +Hyphenator* Hyphenator::loadBinary(const uint8_t* patternData, size_t minPrefix, size_t minSuffix) { Hyphenator* result = new Hyphenator; result->patternData = patternData; + result->minPrefix = minPrefix; + result->minSuffix = minSuffix; return result; } @@ -120,7 +122,7 @@ void Hyphenator::hyphenate(vector* result, const uint16_t* word result->resize(len); const size_t paddedLen = len + 2; // start and stop code each count for 1 if (patternData != nullptr && - (int)len >= MIN_PREFIX + MIN_SUFFIX && paddedLen <= MAX_HYPHENATED_SIZE) { + len >= minPrefix + minSuffix && paddedLen <= MAX_HYPHENATED_SIZE) { uint16_t alpha_codes[MAX_HYPHENATED_SIZE]; const HyphenationType hyphenValue = alphabetLookup(alpha_codes, word, len); if (hyphenValue != HyphenationType::DONT_BREAK) { @@ -307,7 +309,7 @@ void Hyphenator::hyphenateWithNoPatterns(HyphenationType* result, const uint16_t result[i] = hyphenationTypeBasedOnScript(word[i]); } } else if (prevChar == CHAR_MIDDLE_DOT - && MIN_PREFIX < i && i <= len - MIN_SUFFIX + && minPrefix < i && i <= len - minSuffix && ((word[i - 2] == 'l' && word[i] == 'l') || (word[i - 2] == 'L' && word[i] == 'L')) && strcmp(locale.getLanguage(), "ca") == 0) { @@ -392,7 +394,7 @@ void Hyphenator::hyphenateFromCodes(HyphenationType* result, const uint16_t* cod uint32_t link_shift = trie->link_shift; uint32_t link_mask = trie->link_mask; uint32_t pattern_shift = trie->pattern_shift; - size_t maxOffset = len - MIN_SUFFIX - 1; + size_t maxOffset = len - minSuffix - 1; for (size_t i = 0; i < len - 1; i++) { uint32_t node = 0; // index into Trie table for (size_t j = i; j < len; j++) { @@ -414,7 +416,7 @@ void Hyphenator::hyphenateFromCodes(HyphenationType* result, const uint16_t* cod const uint8_t* pat_buf = pattern->buf(pat_entry); int offset = j + 1 - (pat_len + pat_shift); // offset is the index within buffer that lines up with the start of pat_buf - int start = std::max(MIN_PREFIX - offset, 0); + int start = std::max((int)minPrefix - offset, 0); int end = std::min(pat_len, (int)maxOffset - offset); for (int k = start; k < end; k++) { buffer[offset + k] = std::max(buffer[offset + k], pat_buf[k]); @@ -423,8 +425,8 @@ void Hyphenator::hyphenateFromCodes(HyphenationType* result, const uint16_t* cod } } // Since the above calculation does not modify values outside - // [MIN_PREFIX, len - MIN_SUFFIX], they are left as 0 = DONT_BREAK. - for (size_t i = MIN_PREFIX; i < maxOffset; i++) { + // [minPrefix, len - minSuffix], they are left as 0 = DONT_BREAK. + for (size_t i = minPrefix; i < maxOffset; i++) { // Hyphenation opportunities happen when the hyphenation numbers are odd. result[i] = (buffer[i] & 1u) ? hyphenValue : HyphenationType::DONT_BREAK; } diff --git a/engine/src/flutter/tests/perftests/Hyphenator.cpp b/engine/src/flutter/tests/perftests/Hyphenator.cpp index 1e6411f76df..2107e0575ce 100644 --- a/engine/src/flutter/tests/perftests/Hyphenator.cpp +++ b/engine/src/flutter/tests/perftests/Hyphenator.cpp @@ -22,30 +22,34 @@ namespace minikin { const char* enUsHyph = "/system/usr/hyphen-data/hyph-en-us.hyb"; +const int enUsMinPrefix = 2; +const int enUsMinSuffix = 3; const icu::Locale& usLocale = icu::Locale::getUS(); static void BM_Hyphenator_short_word(benchmark::State& state) { - Hyphenator* hyphenator = Hyphenator::loadBinary(readWholeFile(enUsHyph).data()); + Hyphenator* hyphenator = Hyphenator::loadBinary( + readWholeFile(enUsHyph).data(), enUsMinPrefix, enUsMinSuffix); std::vector word = utf8ToUtf16("hyphen"); std::vector result; while (state.KeepRunning()) { hyphenator->hyphenate(&result, word.data(), word.size(), usLocale); } - Hyphenator::loadBinary(nullptr); + Hyphenator::loadBinary(nullptr, 2, 2); } // TODO: Use BENCHMARK_CAPTURE for parametrise. BENCHMARK(BM_Hyphenator_short_word); static void BM_Hyphenator_long_word(benchmark::State& state) { - Hyphenator* hyphenator = Hyphenator::loadBinary(readWholeFile(enUsHyph).data()); + Hyphenator* hyphenator = Hyphenator::loadBinary( + readWholeFile(enUsHyph).data(), enUsMinPrefix, enUsMinSuffix); std::vector word = utf8ToUtf16( "Pneumonoultramicroscopicsilicovolcanoconiosis"); std::vector result; while (state.KeepRunning()) { hyphenator->hyphenate(&result, word.data(), word.size(), usLocale); } - Hyphenator::loadBinary(nullptr); + Hyphenator::loadBinary(nullptr, 2, 2); } // TODO: Use BENCHMARK_CAPTURE for parametrise. diff --git a/engine/src/flutter/tests/unittest/HyphenatorTest.cpp b/engine/src/flutter/tests/unittest/HyphenatorTest.cpp index 6265c2c6c1b..ecd58a2ea08 100644 --- a/engine/src/flutter/tests/unittest/HyphenatorTest.cpp +++ b/engine/src/flutter/tests/unittest/HyphenatorTest.cpp @@ -51,7 +51,7 @@ const uint16_t EN_DASH = 0x2013; // Simple test for US English. This tests "table", which happens to be the in the exceptions list. TEST_F(HyphenatorTest, usEnglishAutomaticHyphenation) { - Hyphenator* hyphenator = Hyphenator::loadBinary(readWholeFile(usHyph).data()); + Hyphenator* hyphenator = Hyphenator::loadBinary(readWholeFile(usHyph).data(), 2, 3); const uint16_t word[] = {'t', 'a', 'b', 'l', 'e'}; std::vector result; hyphenator->hyphenate(&result, word, NELEM(word), usLocale); @@ -65,22 +65,21 @@ TEST_F(HyphenatorTest, usEnglishAutomaticHyphenation) { // Catalan l·l should break as l-/l TEST_F(HyphenatorTest, catalanMiddleDot) { - Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr); - const uint16_t word[] = {'l', 'l', MIDDLE_DOT, 'l', 'l', 'l'}; + Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2); + const uint16_t word[] = {'l', 'l', MIDDLE_DOT, 'l', 'l'}; std::vector result; hyphenator->hyphenate(&result, word, NELEM(word), catalanLocale); - EXPECT_EQ((size_t) 6, result.size()); + EXPECT_EQ((size_t) 5, result.size()); EXPECT_EQ(HyphenationType::DONT_BREAK, result[0]); EXPECT_EQ(HyphenationType::DONT_BREAK, result[1]); EXPECT_EQ(HyphenationType::DONT_BREAK, result[2]); EXPECT_EQ(HyphenationType::BREAK_AND_REPLACE_WITH_HYPHEN, result[3]); EXPECT_EQ(HyphenationType::DONT_BREAK, result[4]); - EXPECT_EQ(HyphenationType::DONT_BREAK, result[5]); } // Catalan l·l should not break if the word is too short. TEST_F(HyphenatorTest, catalanMiddleDotShortWord) { - Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr); + Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2); const uint16_t word[] = {'l', MIDDLE_DOT, 'l'}; std::vector result; hyphenator->hyphenate(&result, word, NELEM(word), catalanLocale); @@ -92,7 +91,7 @@ TEST_F(HyphenatorTest, catalanMiddleDotShortWord) { // If we break on a hyphen in Polish, the hyphen should be repeated on the next line. TEST_F(HyphenatorTest, polishHyphen) { - Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr); + Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2); const uint16_t word[] = {'x', HYPHEN, 'y'}; std::vector result; hyphenator->hyphenate(&result, word, NELEM(word), polishLocale); @@ -104,7 +103,7 @@ TEST_F(HyphenatorTest, polishHyphen) { // If the language is Polish but the script is not Latin, don't use Polish rules for hyphenation. TEST_F(HyphenatorTest, polishHyphenButNonLatinWord) { - Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr); + Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2); const uint16_t word[] = {GREEK_LOWER_ALPHA, HYPHEN, GREEK_LOWER_ALPHA}; std::vector result; hyphenator->hyphenate(&result, word, NELEM(word), polishLocale); @@ -117,7 +116,7 @@ TEST_F(HyphenatorTest, polishHyphenButNonLatinWord) { // Polish en dash doesn't repeat on next line (as far as we know), but just provides a break // opportunity. TEST_F(HyphenatorTest, polishEnDash) { - Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr); + Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2); const uint16_t word[] = {'x', EN_DASH, 'y'}; std::vector result; hyphenator->hyphenate(&result, word, NELEM(word), polishLocale); @@ -129,7 +128,7 @@ TEST_F(HyphenatorTest, polishEnDash) { // In Latin script text, soft hyphens should insert a visible hyphen if broken at. TEST_F(HyphenatorTest, latinSoftHyphen) { - Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr); + Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2); const uint16_t word[] = {'x', SOFT_HYPHEN, 'y'}; std::vector result; hyphenator->hyphenate(&result, word, NELEM(word), usLocale); @@ -141,7 +140,7 @@ TEST_F(HyphenatorTest, latinSoftHyphen) { // Soft hyphens at the beginning of a word are not useful in linebreaking. TEST_F(HyphenatorTest, latinSoftHyphenStartingTheWord) { - Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr); + Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2); const uint16_t word[] = {SOFT_HYPHEN, 'y'}; std::vector result; hyphenator->hyphenate(&result, word, NELEM(word), usLocale); @@ -152,7 +151,7 @@ TEST_F(HyphenatorTest, latinSoftHyphenStartingTheWord) { // In Malayalam script text, soft hyphens should not insert a visible hyphen if broken at. TEST_F(HyphenatorTest, malayalamSoftHyphen) { - Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr); + Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2); const uint16_t word[] = {MALAYALAM_KA, SOFT_HYPHEN, MALAYALAM_KA}; std::vector result; hyphenator->hyphenate(&result, word, NELEM(word), usLocale); @@ -164,7 +163,7 @@ TEST_F(HyphenatorTest, malayalamSoftHyphen) { // In automatically hyphenated Malayalam script text, we should not insert a visible hyphen. TEST_F(HyphenatorTest, malayalamAutomaticHyphenation) { - Hyphenator* hyphenator = Hyphenator::loadBinary(readWholeFile(malayalamHyph).data()); + Hyphenator* hyphenator = Hyphenator::loadBinary(readWholeFile(malayalamHyph).data(), 2, 2); const uint16_t word[] = { MALAYALAM_KA, MALAYALAM_KA, MALAYALAM_KA, MALAYALAM_KA, MALAYALAM_KA}; std::vector result; @@ -173,13 +172,13 @@ TEST_F(HyphenatorTest, malayalamAutomaticHyphenation) { EXPECT_EQ(HyphenationType::DONT_BREAK, result[0]); EXPECT_EQ(HyphenationType::DONT_BREAK, result[1]); EXPECT_EQ(HyphenationType::BREAK_AND_DONT_INSERT_HYPHEN, result[2]); - EXPECT_EQ(HyphenationType::DONT_BREAK, result[3]); + EXPECT_EQ(HyphenationType::BREAK_AND_DONT_INSERT_HYPHEN, result[3]); EXPECT_EQ(HyphenationType::DONT_BREAK, result[4]); } // In Armenian script text, soft hyphens should insert an Armenian hyphen if broken at. TEST_F(HyphenatorTest, aremenianSoftHyphen) { - Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr); + Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2); const uint16_t word[] = {ARMENIAN_AYB, SOFT_HYPHEN, ARMENIAN_AYB}; std::vector result; hyphenator->hyphenate(&result, word, NELEM(word), usLocale); @@ -192,7 +191,7 @@ TEST_F(HyphenatorTest, aremenianSoftHyphen) { // In Hebrew script text, soft hyphens should insert a normal hyphen if broken at, for now. // We may need to change this to maqaf later. TEST_F(HyphenatorTest, hebrewSoftHyphen) { - Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr); + Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2); const uint16_t word[] = {HEBREW_ALEF, SOFT_HYPHEN, HEBREW_ALEF}; std::vector result; hyphenator->hyphenate(&result, word, NELEM(word), usLocale); @@ -205,7 +204,7 @@ TEST_F(HyphenatorTest, hebrewSoftHyphen) { // Soft hyphen between two Arabic letters that join should keep the joining // behavior when broken across lines. TEST_F(HyphenatorTest, arabicSoftHyphenConnecting) { - Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr); + Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2); const uint16_t word[] = {ARABIC_BEH, SOFT_HYPHEN, ARABIC_BEH}; std::vector result; hyphenator->hyphenate(&result, word, NELEM(word), usLocale); @@ -218,7 +217,7 @@ TEST_F(HyphenatorTest, arabicSoftHyphenConnecting) { // Arabic letters may be joining on one side, but if it's the wrong side, we // should use the normal hyphen. TEST_F(HyphenatorTest, arabicSoftHyphenNonConnecting) { - Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr); + Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2); const uint16_t word[] = {ARABIC_ALEF, SOFT_HYPHEN, ARABIC_BEH}; std::vector result; hyphenator->hyphenate(&result, word, NELEM(word), usLocale); @@ -230,7 +229,7 @@ TEST_F(HyphenatorTest, arabicSoftHyphenNonConnecting) { // Skip transparent characters until you find a non-transparent one. TEST_F(HyphenatorTest, arabicSoftHyphenSkipTransparents) { - Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr); + Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2); const uint16_t word[] = {ARABIC_BEH, ARABIC_ZWARAKAY, SOFT_HYPHEN, ARABIC_ZWARAKAY, ARABIC_BEH}; std::vector result; hyphenator->hyphenate(&result, word, NELEM(word), usLocale); @@ -245,7 +244,7 @@ TEST_F(HyphenatorTest, arabicSoftHyphenSkipTransparents) { // Skip transparent characters until you find a non-transparent one. If we get to one end without // finding anything, we are still non-joining. TEST_F(HyphenatorTest, arabicSoftHyphenTransparentsAtEnd) { - Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr); + Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2); const uint16_t word[] = {ARABIC_BEH, ARABIC_ZWARAKAY, SOFT_HYPHEN, ARABIC_ZWARAKAY}; std::vector result; hyphenator->hyphenate(&result, word, NELEM(word), usLocale); @@ -259,7 +258,7 @@ TEST_F(HyphenatorTest, arabicSoftHyphenTransparentsAtEnd) { // Skip transparent characters until you find a non-transparent one. If we get to one end without // finding anything, we are still non-joining. TEST_F(HyphenatorTest, arabicSoftHyphenTransparentsAtStart) { - Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr); + Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2); const uint16_t word[] = {ARABIC_ZWARAKAY, SOFT_HYPHEN, ARABIC_ZWARAKAY, ARABIC_BEH}; std::vector result; hyphenator->hyphenate(&result, word, NELEM(word), usLocale); @@ -272,7 +271,7 @@ TEST_F(HyphenatorTest, arabicSoftHyphenTransparentsAtStart) { // In Unified Canadian Aboriginal script (UCAS) text, soft hyphens should insert a UCAS hyphen. TEST_F(HyphenatorTest, ucasSoftHyphen) { - Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr); + Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2); const uint16_t word[] = {UCAS_E, SOFT_HYPHEN, UCAS_E}; std::vector result; hyphenator->hyphenate(&result, word, NELEM(word), usLocale); @@ -285,7 +284,7 @@ TEST_F(HyphenatorTest, ucasSoftHyphen) { // Presently, soft hyphen looks at the character after it to determine hyphenation type. This is a // little arbitrary, but let's test it anyway. TEST_F(HyphenatorTest, mixedScriptSoftHyphen) { - Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr); + Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2); const uint16_t word[] = {'a', SOFT_HYPHEN, UCAS_E}; std::vector result; hyphenator->hyphenate(&result, word, NELEM(word), usLocale); @@ -297,7 +296,7 @@ TEST_F(HyphenatorTest, mixedScriptSoftHyphen) { // Hard hyphens provide a breaking opportunity with nothing extra inserted. TEST_F(HyphenatorTest, hardHyphen) { - Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr); + Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2); const uint16_t word[] = {'x', HYPHEN, 'y'}; std::vector result; hyphenator->hyphenate(&result, word, NELEM(word), usLocale); @@ -309,7 +308,7 @@ TEST_F(HyphenatorTest, hardHyphen) { // Hyphen-minuses also provide a breaking opportunity with nothing extra inserted. TEST_F(HyphenatorTest, hyphenMinus) { - Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr); + Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2); const uint16_t word[] = {'x', HYPHEN_MINUS, 'y'}; std::vector result; hyphenator->hyphenate(&result, word, NELEM(word), usLocale); @@ -322,7 +321,7 @@ TEST_F(HyphenatorTest, hyphenMinus) { // If the word starts with a hard hyphen or hyphen-minus, it doesn't make sense to break // it at that point. TEST_F(HyphenatorTest, startingHyphenMinus) { - Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr); + Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2); const uint16_t word[] = {HYPHEN_MINUS, 'y'}; std::vector result; hyphenator->hyphenate(&result, word, NELEM(word), usLocale);