DEV/flutter_flutter
1
0
Fork 0
mirror of https://github.com/flutter/flutter.git synced 2026-02-20 02:29:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

Customizable min suffix/prefix length for hyphenation in Minikin

Browse Source 
With this change, different languages can have a different minimum
length for suffix and prefixes when hyphenating. Previously, the
defaults used for English, 2 and 3, were used for every language.

Bug: 35712376
Test: Minikin unit tests were updated and the pass
Change-Id: Iffaf11c6b208c57d28d45b17246e177572dc1210
This commit is contained in:
Roozbeh Pournader 2017-02-27 18:54:01 -08:00
parent 0b451587cc
commit 068c7ba2ea
5 changed files with 47 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
engine/src/flutter/app/HyphTool.cpp
View File

@ -11,7 +11,7 @@
using minikin::HyphenationType;
using minikin::Hyphenator;
Hyphenator* loadHybFile(const char* fn) {
Hyphenator* loadHybFile(const char* fn, int minPrefix, int minSuffix) {
struct stat statbuf;
int status = stat(fn, &statbuf);
if (status < 0) {
@ -32,11 +32,11 @@ Hyphenator* loadHybFile(const char* fn) {
delete[] buf;
return nullptr;
}
return Hyphenator::loadBinary(buf);
return Hyphenator::loadBinary(buf, minPrefix, minSuffix);
}
int main(int argc, char** argv) {
Hyphenator* hyph = loadHybFile("/tmp/en.hyb"); // should also be configurable
Hyphenator* hyph = loadHybFile("/tmp/en.hyb", 2, 3); // should also be configurable
std::vector<HyphenationType> result;
std::vector<uint16_t> word;
if (argc < 2) {

8
engine/src/flutter/include/minikin/Hyphenator.h
View File

@ -136,7 +136,7 @@ public:
// at least as long as the Hyphenator object.
// Note: nullptr is valid input, in which case the hyphenator only processes soft hyphens.
static Hyphenator* loadBinary(const uint8_t* patternData);
static Hyphenator* loadBinary(const uint8_t* patternData, size_t minPrefix, size_t minSuffix);
private:
// apply various hyphenation rules including hard and soft hyphens, ignoring patterns
@ -153,17 +153,13 @@ private:
void hyphenateFromCodes(HyphenationType* result, const uint16_t* codes, size_t len,
HyphenationType hyphenValue);
// TODO: these should become parameters, as they might vary by locale, screen size, and
// possibly explicit user control.
static const int MIN_PREFIX = 2;
static const int MIN_SUFFIX = 3;
// See also LONGEST_HYPHENATED_WORD in LineBreaker.cpp. Here the constant is used so
// that temporary buffers can be stack-allocated without waste, which is a slightly
// different use case. It measures UTF-16 code units.
static const size_t MAX_HYPHENATED_SIZE = 64;
const uint8_t* patternData;
size_t minPrefix, minSuffix;
// accessors for binary data
const Header* getHeader() const {

16
engine/src/flutter/libs/minikin/Hyphenator.cpp
View File

@ -108,9 +108,11 @@ struct Header {
}
};
Hyphenator* Hyphenator::loadBinary(const uint8_t* patternData) {
Hyphenator* Hyphenator::loadBinary(const uint8_t* patternData, size_t minPrefix, size_t minSuffix) {
Hyphenator* result = new Hyphenator;
result->patternData = patternData;
result->minPrefix = minPrefix;
result->minSuffix = minSuffix;
return result;
}
@ -120,7 +122,7 @@ void Hyphenator::hyphenate(vector<HyphenationType>* result, const uint16_t* word
result->resize(len);
const size_t paddedLen = len + 2; // start and stop code each count for 1
if (patternData != nullptr &&
(int)len >= MIN_PREFIX + MIN_SUFFIX && paddedLen <= MAX_HYPHENATED_SIZE) {
len >= minPrefix + minSuffix && paddedLen <= MAX_HYPHENATED_SIZE) {
uint16_t alpha_codes[MAX_HYPHENATED_SIZE];
const HyphenationType hyphenValue = alphabetLookup(alpha_codes, word, len);
if (hyphenValue != HyphenationType::DONT_BREAK) {
@ -307,7 +309,7 @@ void Hyphenator::hyphenateWithNoPatterns(HyphenationType* result, const uint16_t
result[i] = hyphenationTypeBasedOnScript(word[i]);
}
} else if (prevChar == CHAR_MIDDLE_DOT
&& MIN_PREFIX < i && i <= len - MIN_SUFFIX
&& minPrefix < i && i <= len - minSuffix
&& ((word[i - 2] == 'l' && word[i] == 'l')
|| (word[i - 2] == 'L' && word[i] == 'L'))
&& strcmp(locale.getLanguage(), "ca") == 0) {
@ -392,7 +394,7 @@ void Hyphenator::hyphenateFromCodes(HyphenationType* result, const uint16_t* cod
uint32_t link_shift = trie->link_shift;
uint32_t link_mask = trie->link_mask;
uint32_t pattern_shift = trie->pattern_shift;
size_t maxOffset = len - MIN_SUFFIX - 1;
size_t maxOffset = len - minSuffix - 1;
for (size_t i = 0; i < len - 1; i++) {
uint32_t node = 0; // index into Trie table
for (size_t j = i; j < len; j++) {
@ -414,7 +416,7 @@ void Hyphenator::hyphenateFromCodes(HyphenationType* result, const uint16_t* cod
const uint8_t* pat_buf = pattern->buf(pat_entry);
int offset = j + 1 - (pat_len + pat_shift);
// offset is the index within buffer that lines up with the start of pat_buf
int start = std::max(MIN_PREFIX - offset, 0);
int start = std::max((int)minPrefix - offset, 0);
int end = std::min(pat_len, (int)maxOffset - offset);
for (int k = start; k < end; k++) {
buffer[offset + k] = std::max(buffer[offset + k], pat_buf[k]);
@ -423,8 +425,8 @@ void Hyphenator::hyphenateFromCodes(HyphenationType* result, const uint16_t* cod
}
}
// Since the above calculation does not modify values outside
// [MIN_PREFIX, len - MIN_SUFFIX], they are left as 0 = DONT_BREAK.
for (size_t i = MIN_PREFIX; i < maxOffset; i++) {
// [minPrefix, len - minSuffix], they are left as 0 = DONT_BREAK.
for (size_t i = minPrefix; i < maxOffset; i++) {
// Hyphenation opportunities happen when the hyphenation numbers are odd.
result[i] = (buffer[i] & 1u) ? hyphenValue : HyphenationType::DONT_BREAK;
}

12
engine/src/flutter/tests/perftests/Hyphenator.cpp
View File

@ -22,30 +22,34 @@
namespace minikin {
const char* enUsHyph = "/system/usr/hyphen-data/hyph-en-us.hyb";
const int enUsMinPrefix = 2;
const int enUsMinSuffix = 3;
const icu::Locale& usLocale = icu::Locale::getUS();
static void BM_Hyphenator_short_word(benchmark::State& state) {
Hyphenator* hyphenator = Hyphenator::loadBinary(readWholeFile(enUsHyph).data());
Hyphenator* hyphenator = Hyphenator::loadBinary(
readWholeFile(enUsHyph).data(), enUsMinPrefix, enUsMinSuffix);
std::vector<uint16_t> word = utf8ToUtf16("hyphen");
std::vector<HyphenationType> result;
while (state.KeepRunning()) {
hyphenator->hyphenate(&result, word.data(), word.size(), usLocale);
}
Hyphenator::loadBinary(nullptr);
Hyphenator::loadBinary(nullptr, 2, 2);
}
// TODO: Use BENCHMARK_CAPTURE for parametrise.
BENCHMARK(BM_Hyphenator_short_word);
static void BM_Hyphenator_long_word(benchmark::State& state) {
Hyphenator* hyphenator = Hyphenator::loadBinary(readWholeFile(enUsHyph).data());
Hyphenator* hyphenator = Hyphenator::loadBinary(
readWholeFile(enUsHyph).data(), enUsMinPrefix, enUsMinSuffix);
std::vector<uint16_t> word = utf8ToUtf16(
"Pneumonoultramicroscopicsilicovolcanoconiosis");
std::vector<HyphenationType> result;
while (state.KeepRunning()) {
hyphenator->hyphenate(&result, word.data(), word.size(), usLocale);
}
Hyphenator::loadBinary(nullptr);
Hyphenator::loadBinary(nullptr, 2, 2);
}
// TODO: Use BENCHMARK_CAPTURE for parametrise.

51
engine/src/flutter/tests/unittest/HyphenatorTest.cpp
View File

@ -51,7 +51,7 @@ const uint16_t EN_DASH = 0x2013;
// Simple test for US English. This tests "table", which happens to be the in the exceptions list.
TEST_F(HyphenatorTest, usEnglishAutomaticHyphenation) {
Hyphenator* hyphenator = Hyphenator::loadBinary(readWholeFile(usHyph).data());
Hyphenator* hyphenator = Hyphenator::loadBinary(readWholeFile(usHyph).data(), 2, 3);
const uint16_t word[] = {'t', 'a', 'b', 'l', 'e'};
std::vector<HyphenationType> result;
hyphenator->hyphenate(&result, word, NELEM(word), usLocale);
@ -65,22 +65,21 @@ TEST_F(HyphenatorTest, usEnglishAutomaticHyphenation) {
// Catalan l·l should break as l-/l
TEST_F(HyphenatorTest, catalanMiddleDot) {
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr);
const uint16_t word[] = {'l', 'l', MIDDLE_DOT, 'l', 'l', 'l'};
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2);
const uint16_t word[] = {'l', 'l', MIDDLE_DOT, 'l', 'l'};
std::vector<HyphenationType> result;
hyphenator->hyphenate(&result, word, NELEM(word), catalanLocale);
EXPECT_EQ((size_t) 6, result.size());
EXPECT_EQ((size_t) 5, result.size());
EXPECT_EQ(HyphenationType::DONT_BREAK, result[0]);
EXPECT_EQ(HyphenationType::DONT_BREAK, result[1]);
EXPECT_EQ(HyphenationType::DONT_BREAK, result[2]);
EXPECT_EQ(HyphenationType::BREAK_AND_REPLACE_WITH_HYPHEN, result[3]);
EXPECT_EQ(HyphenationType::DONT_BREAK, result[4]);
EXPECT_EQ(HyphenationType::DONT_BREAK, result[5]);
}
// Catalan l·l should not break if the word is too short.
TEST_F(HyphenatorTest, catalanMiddleDotShortWord) {
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr);
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2);
const uint16_t word[] = {'l', MIDDLE_DOT, 'l'};
std::vector<HyphenationType> result;
hyphenator->hyphenate(&result, word, NELEM(word), catalanLocale);
@ -92,7 +91,7 @@ TEST_F(HyphenatorTest, catalanMiddleDotShortWord) {
// If we break on a hyphen in Polish, the hyphen should be repeated on the next line.
TEST_F(HyphenatorTest, polishHyphen) {
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr);
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2);
const uint16_t word[] = {'x', HYPHEN, 'y'};
std::vector<HyphenationType> result;
hyphenator->hyphenate(&result, word, NELEM(word), polishLocale);
@ -104,7 +103,7 @@ TEST_F(HyphenatorTest, polishHyphen) {
// If the language is Polish but the script is not Latin, don't use Polish rules for hyphenation.
TEST_F(HyphenatorTest, polishHyphenButNonLatinWord) {
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr);
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2);
const uint16_t word[] = {GREEK_LOWER_ALPHA, HYPHEN, GREEK_LOWER_ALPHA};
std::vector<HyphenationType> result;
hyphenator->hyphenate(&result, word, NELEM(word), polishLocale);
@ -117,7 +116,7 @@ TEST_F(HyphenatorTest, polishHyphenButNonLatinWord) {
// Polish en dash doesn't repeat on next line (as far as we know), but just provides a break
// opportunity.
TEST_F(HyphenatorTest, polishEnDash) {
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr);
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2);
const uint16_t word[] = {'x', EN_DASH, 'y'};
std::vector<HyphenationType> result;
hyphenator->hyphenate(&result, word, NELEM(word), polishLocale);
@ -129,7 +128,7 @@ TEST_F(HyphenatorTest, polishEnDash) {
// In Latin script text, soft hyphens should insert a visible hyphen if broken at.
TEST_F(HyphenatorTest, latinSoftHyphen) {
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr);
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2);
const uint16_t word[] = {'x', SOFT_HYPHEN, 'y'};
std::vector<HyphenationType> result;
hyphenator->hyphenate(&result, word, NELEM(word), usLocale);
@ -141,7 +140,7 @@ TEST_F(HyphenatorTest, latinSoftHyphen) {
// Soft hyphens at the beginning of a word are not useful in linebreaking.
TEST_F(HyphenatorTest, latinSoftHyphenStartingTheWord) {
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr);
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2);
const uint16_t word[] = {SOFT_HYPHEN, 'y'};
std::vector<HyphenationType> result;
hyphenator->hyphenate(&result, word, NELEM(word), usLocale);
@ -152,7 +151,7 @@ TEST_F(HyphenatorTest, latinSoftHyphenStartingTheWord) {
// In Malayalam script text, soft hyphens should not insert a visible hyphen if broken at.
TEST_F(HyphenatorTest, malayalamSoftHyphen) {
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr);
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2);
const uint16_t word[] = {MALAYALAM_KA, SOFT_HYPHEN, MALAYALAM_KA};
std::vector<HyphenationType> result;
hyphenator->hyphenate(&result, word, NELEM(word), usLocale);
@ -164,7 +163,7 @@ TEST_F(HyphenatorTest, malayalamSoftHyphen) {
// In automatically hyphenated Malayalam script text, we should not insert a visible hyphen.
TEST_F(HyphenatorTest, malayalamAutomaticHyphenation) {
Hyphenator* hyphenator = Hyphenator::loadBinary(readWholeFile(malayalamHyph).data());
Hyphenator* hyphenator = Hyphenator::loadBinary(readWholeFile(malayalamHyph).data(), 2, 2);
const uint16_t word[] = {
MALAYALAM_KA, MALAYALAM_KA, MALAYALAM_KA, MALAYALAM_KA, MALAYALAM_KA};
std::vector<HyphenationType> result;
@ -173,13 +172,13 @@ TEST_F(HyphenatorTest, malayalamAutomaticHyphenation) {
EXPECT_EQ(HyphenationType::DONT_BREAK, result[0]);
EXPECT_EQ(HyphenationType::DONT_BREAK, result[1]);
EXPECT_EQ(HyphenationType::BREAK_AND_DONT_INSERT_HYPHEN, result[2]);
EXPECT_EQ(HyphenationType::DONT_BREAK, result[3]);
EXPECT_EQ(HyphenationType::BREAK_AND_DONT_INSERT_HYPHEN, result[3]);
EXPECT_EQ(HyphenationType::DONT_BREAK, result[4]);
}
// In Armenian script text, soft hyphens should insert an Armenian hyphen if broken at.
TEST_F(HyphenatorTest, aremenianSoftHyphen) {
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr);
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2);
const uint16_t word[] = {ARMENIAN_AYB, SOFT_HYPHEN, ARMENIAN_AYB};
std::vector<HyphenationType> result;
hyphenator->hyphenate(&result, word, NELEM(word), usLocale);
@ -192,7 +191,7 @@ TEST_F(HyphenatorTest, aremenianSoftHyphen) {
// In Hebrew script text, soft hyphens should insert a normal hyphen if broken at, for now.
// We may need to change this to maqaf later.
TEST_F(HyphenatorTest, hebrewSoftHyphen) {
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr);
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2);
const uint16_t word[] = {HEBREW_ALEF, SOFT_HYPHEN, HEBREW_ALEF};
std::vector<HyphenationType> result;
hyphenator->hyphenate(&result, word, NELEM(word), usLocale);
@ -205,7 +204,7 @@ TEST_F(HyphenatorTest, hebrewSoftHyphen) {
// Soft hyphen between two Arabic letters that join should keep the joining
// behavior when broken across lines.
TEST_F(HyphenatorTest, arabicSoftHyphenConnecting) {
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr);
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2);
const uint16_t word[] = {ARABIC_BEH, SOFT_HYPHEN, ARABIC_BEH};
std::vector<HyphenationType> result;
hyphenator->hyphenate(&result, word, NELEM(word), usLocale);
@ -218,7 +217,7 @@ TEST_F(HyphenatorTest, arabicSoftHyphenConnecting) {
// Arabic letters may be joining on one side, but if it's the wrong side, we
// should use the normal hyphen.
TEST_F(HyphenatorTest, arabicSoftHyphenNonConnecting) {
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr);
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2);
const uint16_t word[] = {ARABIC_ALEF, SOFT_HYPHEN, ARABIC_BEH};
std::vector<HyphenationType> result;
hyphenator->hyphenate(&result, word, NELEM(word), usLocale);
@ -230,7 +229,7 @@ TEST_F(HyphenatorTest, arabicSoftHyphenNonConnecting) {
// Skip transparent characters until you find a non-transparent one.
TEST_F(HyphenatorTest, arabicSoftHyphenSkipTransparents) {
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr);
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2);
const uint16_t word[] = {ARABIC_BEH, ARABIC_ZWARAKAY, SOFT_HYPHEN, ARABIC_ZWARAKAY, ARABIC_BEH};
std::vector<HyphenationType> result;
hyphenator->hyphenate(&result, word, NELEM(word), usLocale);
@ -245,7 +244,7 @@ TEST_F(HyphenatorTest, arabicSoftHyphenSkipTransparents) {
// Skip transparent characters until you find a non-transparent one. If we get to one end without
// finding anything, we are still non-joining.
TEST_F(HyphenatorTest, arabicSoftHyphenTransparentsAtEnd) {
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr);
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2);
const uint16_t word[] = {ARABIC_BEH, ARABIC_ZWARAKAY, SOFT_HYPHEN, ARABIC_ZWARAKAY};
std::vector<HyphenationType> result;
hyphenator->hyphenate(&result, word, NELEM(word), usLocale);
@ -259,7 +258,7 @@ TEST_F(HyphenatorTest, arabicSoftHyphenTransparentsAtEnd) {
// Skip transparent characters until you find a non-transparent one. If we get to one end without
// finding anything, we are still non-joining.
TEST_F(HyphenatorTest, arabicSoftHyphenTransparentsAtStart) {
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr);
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2);
const uint16_t word[] = {ARABIC_ZWARAKAY, SOFT_HYPHEN, ARABIC_ZWARAKAY, ARABIC_BEH};
std::vector<HyphenationType> result;
hyphenator->hyphenate(&result, word, NELEM(word), usLocale);
@ -272,7 +271,7 @@ TEST_F(HyphenatorTest, arabicSoftHyphenTransparentsAtStart) {
// In Unified Canadian Aboriginal script (UCAS) text, soft hyphens should insert a UCAS hyphen.
TEST_F(HyphenatorTest, ucasSoftHyphen) {
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr);
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2);
const uint16_t word[] = {UCAS_E, SOFT_HYPHEN, UCAS_E};
std::vector<HyphenationType> result;
hyphenator->hyphenate(&result, word, NELEM(word), usLocale);
@ -285,7 +284,7 @@ TEST_F(HyphenatorTest, ucasSoftHyphen) {
// Presently, soft hyphen looks at the character after it to determine hyphenation type. This is a
// little arbitrary, but let's test it anyway.
TEST_F(HyphenatorTest, mixedScriptSoftHyphen) {
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr);
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2);
const uint16_t word[] = {'a', SOFT_HYPHEN, UCAS_E};
std::vector<HyphenationType> result;
hyphenator->hyphenate(&result, word, NELEM(word), usLocale);
@ -297,7 +296,7 @@ TEST_F(HyphenatorTest, mixedScriptSoftHyphen) {
// Hard hyphens provide a breaking opportunity with nothing extra inserted.
TEST_F(HyphenatorTest, hardHyphen) {
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr);
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2);
const uint16_t word[] = {'x', HYPHEN, 'y'};
std::vector<HyphenationType> result;
hyphenator->hyphenate(&result, word, NELEM(word), usLocale);
@ -309,7 +308,7 @@ TEST_F(HyphenatorTest, hardHyphen) {
// Hyphen-minuses also provide a breaking opportunity with nothing extra inserted.
TEST_F(HyphenatorTest, hyphenMinus) {
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr);
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2);
const uint16_t word[] = {'x', HYPHEN_MINUS, 'y'};
std::vector<HyphenationType> result;
hyphenator->hyphenate(&result, word, NELEM(word), usLocale);
@ -322,7 +321,7 @@ TEST_F(HyphenatorTest, hyphenMinus) {
// If the word starts with a hard hyphen or hyphen-minus, it doesn't make sense to break
// it at that point.
TEST_F(HyphenatorTest, startingHyphenMinus) {
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr);
Hyphenator* hyphenator = Hyphenator::loadBinary(nullptr, 2, 2);
const uint16_t word[] = {HYPHEN_MINUS, 'y'};
std::vector<HyphenationType> result;
hyphenator->hyphenate(&result, word, NELEM(word), usLocale);