eastasianwidth.go

@@ -1,13 +1,13 @@-package uniseg- // Code generated via go generate from gen_properties.go. DO NOT EDIT. +package uniseg+ // eastAsianWidth are taken from // https://www.unicode.org/Public/14.0.0/ucd/EastAsianWidth.txt // and // https://unicode.org/Public/14.0.0/ucd/emoji/emoji-data.txt // ("Extended_Pictographic" only)-// on September 10, 2022. See https://www.unicode.org/license.html for the Unicode+// on July 17, 2023. See https://www.unicode.org/license.html for the Unicode // license agreement. var eastAsianWidth = [][3]int{ 	{0x0000, 0x001F, prN},     // Cc    [32] <control-0000>..<control-001F>

emojipresentation.go

@@ -1,13 +1,13 @@-package uniseg- // Code generated via go generate from gen_properties.go. DO NOT EDIT. +package uniseg+ // emojiPresentation are taken from // // and // https://unicode.org/Public/14.0.0/ucd/emoji/emoji-data.txt // ("Extended_Pictographic" only)-// on September 10, 2022. See https://www.unicode.org/license.html for the Unicode+// on July 17, 2023. See https://www.unicode.org/license.html for the Unicode // license agreement. var emojiPresentation = [][3]int{ 	{0x231A, 0x231B, prEmojiPresentation},   // E0.6   [2] (⌚..⌛)    watch..hourglass done

gen_breaktest.go

@@ -76,9 +76,9 @@ func parse(url string) ([]byte, error) {  	buf := new(bytes.Buffer) 	buf.Grow(120 << 10)-	buf.WriteString(`package uniseg+	buf.WriteString(`// Code generated via go generate from gen_breaktest.go. DO NOT EDIT. -// Code generated via go generate from gen_breaktest.go. DO NOT EDIT.+package uniseg  // ` + os.Args[3] + ` are Grapheme testcases taken from // ` + url + `@@ -136,7 +136,9 @@ var ( // // E.g. for the input b="÷ 0020 × 0308 ÷ 1F1E6 ÷" // it will append-//     "\u0020\u0308\U0001F1E6"+//+//	"\u0020\u0308\U0001F1E6"+// // and "[][]rune{{0x0020,0x0308},{0x1F1E6},}" // to orig and exp respectively. //

gen_properties.go

@@ -178,6 +178,11 @@ func parse(propertyURL, emojiProperty string, includeGeneralCategory bool) (stri 		} 	} +	// Avoid overflow during binary search.+	if len(properties) >= 1<<31 {+		return "", errors.New("too many properties")+	}+ 	// Sort properties. 	sort.Slice(properties, func(i, j int) bool { 		left, _ := strconv.ParseUint(properties[i][0], 16, 64)@@ -200,9 +205,9 @@ func parse(propertyURL, emojiProperty string, includeGeneralCategory bool) (stri // ` + emojiURL + ` // ("Extended_Pictographic" only)` 	}-	buf.WriteString(`package uniseg+	buf.WriteString(`// Code generated via go generate from gen_properties.go. DO NOT EDIT. -// Code generated via go generate from gen_properties.go. DO NOT EDIT.+package uniseg  // ` + os.Args[3] + ` are taken from // ` + propertyURL + emojiComment + `

grapheme_test.go

@@ -6,6 +6,8 @@ import (  const benchmarkStr = "This is 🏳️‍🌈, a test string ツ for grapheme cluster testing. 🏋🏽‍♀️🙂🙂 It's only relevant for benchmark tests." +var benchmarkBytes = []byte(benchmarkStr)+ // Variables to avoid compiler optimizations. var resultRunes []rune @@ -520,10 +522,10 @@ func BenchmarkGraphemesClass(b *testing.B) {  // Benchmark the use of the Graphemes function for byte slices. func BenchmarkGraphemesFunctionBytes(b *testing.B) {-	str := []byte(benchmarkStr) 	for i := 0; i < b.N; i++ { 		var c []byte 		state := -1+		str := benchmarkBytes 		for len(str) > 0 { 			c, str, _, state = FirstGraphemeCluster(str, state) 			resultRunes = []rune(string(c))@@ -533,10 +535,10 @@ func BenchmarkGraphemesFunctionBytes(b *testing.B) {  // Benchmark the use of the Graphemes function for strings. func BenchmarkGraphemesFunctionString(b *testing.B) {-	str := benchmarkStr 	for i := 0; i < b.N; i++ { 		var c string 		state := -1+		str := benchmarkStr 		for len(str) > 0 { 			c, str, _, state = FirstGraphemeClusterInString(str, state) 			resultRunes = []rune(c)

graphemebreak_test.go

@@ -1,10 +1,10 @@-package uniseg- // Code generated via go generate from gen_breaktest.go. DO NOT EDIT. +package uniseg+ // graphemeBreakTestCases are Grapheme testcases taken from // https://www.unicode.org/Public/14.0.0/ucd/auxiliary/GraphemeBreakTest.txt-// on September 10, 2022. See+// on July 17, 2023. See // https://www.unicode.org/license.html for the Unicode license agreement. var graphemeBreakTestCases = []testCase{ 	{original: "\u0020\u0020", expected: [][]rune{{0x0020}, {0x0020}}},                                                                                 // ÷ [0.2] SPACE (Other) ÷ [999.0] SPACE (Other) ÷ [0.3]

graphemeproperties.go

@@ -1,13 +1,13 @@-package uniseg- // Code generated via go generate from gen_properties.go. DO NOT EDIT. +package uniseg+ // graphemeCodePoints are taken from // https://www.unicode.org/Public/14.0.0/ucd/auxiliary/GraphemeBreakProperty.txt // and // https://unicode.org/Public/14.0.0/ucd/emoji/emoji-data.txt // ("Extended_Pictographic" only)-// on September 10, 2022. See https://www.unicode.org/license.html for the Unicode+// on July 17, 2023. See https://www.unicode.org/license.html for the Unicode // license agreement. var graphemeCodePoints = [][3]int{ 	{0x0000, 0x0009, prControl},                // Cc  [10] <control-0000>..<control-0009>

line.go

@@ -123,12 +123,12 @@ func FirstLineSegmentInString(str string, state int) (segment, rest string, must func HasTrailingLineBreak(b []byte) bool { 	r, _ := utf8.DecodeLastRune(b) 	property, _ := propertyWithGenCat(lineBreakCodePoints, r)-	return property == lbBK || property == lbCR || property == lbLF || property == lbNL+	return property == prBK || property == prCR || property == prLF || property == prNL }  // HasTrailingLineBreakInString is like [HasTrailingLineBreak] but for a string. func HasTrailingLineBreakInString(str string) bool { 	r, _ := utf8.DecodeLastRuneInString(str) 	property, _ := propertyWithGenCat(lineBreakCodePoints, r)-	return property == lbBK || property == lbCR || property == lbLF || property == lbNL+	return property == prBK || property == prCR || property == prLF || property == prNL }

line_test.go

@@ -168,12 +168,43 @@ func TestLineCasesString(t *testing.T) { 	} } +var hasTrailingLineBreakTestCases = []struct {+	input string+	want  bool+}{+	{"\v", true},     // prBK+	{"\r", true},     // prCR+	{"\n", true},     // prLF+	{"\u0085", true}, // prNL+	{" ", false},+	{"A", false},+	{"", false},+}++func TestHasTrailingLineBreak(t *testing.T) {+	for _, tt := range hasTrailingLineBreakTestCases {+		got := HasTrailingLineBreak([]byte(tt.input))+		if got != tt.want {+			t.Errorf("HasTrailingLineBreak(%q) = %v, want %v", tt.input, got, tt.want)+		}+	}+}++func TestHasTrailingLineBreakInString(t *testing.T) {+	for _, tt := range hasTrailingLineBreakTestCases {+		got := HasTrailingLineBreakInString(tt.input)+		if got != tt.want {+			t.Errorf("HasTrailingLineBreak(%q) = %v, want %v", tt.input, got, tt.want)+		}+	}+}+ // Benchmark the use of the line break function for byte slices. func BenchmarkLineFunctionBytes(b *testing.B) {-	str := []byte(benchmarkStr) 	for i := 0; i < b.N; i++ { 		var c []byte 		state := -1+		str := benchmarkBytes 		for len(str) > 0 { 			c, str, _, state = FirstLineSegment(str, state) 			resultRunes = []rune(string(c))@@ -183,10 +214,10 @@ func BenchmarkLineFunctionBytes(b *testing.B) {  // Benchmark the use of the line break function for strings. func BenchmarkLineFunctionString(b *testing.B) {-	str := benchmarkStr 	for i := 0; i < b.N; i++ { 		var c string 		state := -1+		str := benchmarkStr 		for len(str) > 0 { 			c, str, _, state = FirstLineSegmentInString(str, state) 			resultRunes = []rune(c)

linebreak_test.go

@@ -1,10 +1,10 @@-package uniseg- // Code generated via go generate from gen_breaktest.go. DO NOT EDIT. +package uniseg+ // lineBreakTestCases are Grapheme testcases taken from // https://www.unicode.org/Public/14.0.0/ucd/auxiliary/LineBreakTest.txt-// on September 10, 2022. See+// on July 17, 2023. See // https://www.unicode.org/license.html for the Unicode license agreement. var lineBreakTestCases = []testCase{ 	{original: "\u0023\u0023", expected: [][]rune{{0x0023, 0x0023}}},                                                                                                                               // × [0.3] NUMBER SIGN (AL) × [28.0] NUMBER SIGN (AL) ÷ [0.3]

lineproperties.go

@@ -1,13 +1,13 @@-package uniseg- // Code generated via go generate from gen_properties.go. DO NOT EDIT. +package uniseg+ // lineBreakCodePoints are taken from // https://www.unicode.org/Public/14.0.0/ucd/LineBreak.txt // and // https://unicode.org/Public/14.0.0/ucd/emoji/emoji-data.txt // ("Extended_Pictographic" only)-// on September 10, 2022. See https://www.unicode.org/license.html for the Unicode+// on July 17, 2023. See https://www.unicode.org/license.html for the Unicode // license agreement. var lineBreakCodePoints = [][4]int{ 	{0x0000, 0x0008, prCM, gcCc},     //     [9] <control-0000>..<control-0008>

sentence_test.go

@@ -148,10 +148,10 @@ func TestSentenceCasesString(t *testing.T) {  // Benchmark the use of the sentence break function for byte slices. func BenchmarkSentenceFunctionBytes(b *testing.B) {-	str := []byte(benchmarkStr) 	for i := 0; i < b.N; i++ { 		var c []byte 		state := -1+		str := benchmarkBytes 		for len(str) > 0 { 			c, str, state = FirstSentence(str, state) 			resultRunes = []rune(string(c))@@ -161,10 +161,10 @@ func BenchmarkSentenceFunctionBytes(b *testing.B) {  // Benchmark the use of the sentence break function for strings. func BenchmarkSentenceFunctionString(b *testing.B) {-	str := benchmarkStr 	for i := 0; i < b.N; i++ { 		var c string 		state := -1+		str := benchmarkStr 		for len(str) > 0 { 			c, str, state = FirstSentenceInString(str, state) 			resultRunes = []rune(c)

sentencebreak_test.go

@@ -1,10 +1,10 @@-package uniseg- // Code generated via go generate from gen_breaktest.go. DO NOT EDIT. +package uniseg+ // sentenceBreakTestCases are Grapheme testcases taken from // https://www.unicode.org/Public/14.0.0/ucd/auxiliary/SentenceBreakTest.txt-// on September 10, 2022. See+// on July 17, 2023. See // https://www.unicode.org/license.html for the Unicode license agreement. var sentenceBreakTestCases = []testCase{ 	{original: "\u0001\u0001", expected: [][]rune{{0x0001, 0x0001}}},                                               // ÷ [0.2] <START OF HEADING> (Other) × [998.0] <START OF HEADING> (Other) ÷ [0.3]

sentenceproperties.go

@@ -1,13 +1,13 @@-package uniseg- // Code generated via go generate from gen_properties.go. DO NOT EDIT. +package uniseg+ // sentenceBreakCodePoints are taken from // https://www.unicode.org/Public/14.0.0/ucd/auxiliary/SentenceBreakProperty.txt // and // https://unicode.org/Public/14.0.0/ucd/emoji/emoji-data.txt // ("Extended_Pictographic" only)-// on September 10, 2022. See https://www.unicode.org/license.html for the Unicode+// on July 17, 2023. See https://www.unicode.org/license.html for the Unicode // license agreement. var sentenceBreakCodePoints = [][3]int{ 	{0x0009, 0x0009, prSp},        // Cc       <control-0009>

step_test.go

@@ -460,10 +460,10 @@ func TestStepStringSentence(t *testing.T) {  // Benchmark the use of the [Step] function. func BenchmarkStepBytes(b *testing.B) {-	str := []byte(benchmarkStr) 	for i := 0; i < b.N; i++ { 		var c []byte 		state := -1+		str := benchmarkBytes 		for len(str) > 0 { 			c, str, _, state = Step(str, state) 			resultRunes = []rune(string(c))@@ -473,10 +473,10 @@ func BenchmarkStepBytes(b *testing.B) {  // Benchmark the use of the StepString() function. func BenchmarkStepString(b *testing.B) {-	str := benchmarkStr 	for i := 0; i < b.N; i++ { 		var c string 		state := -1+		str := benchmarkStr 		for len(str) > 0 { 			c, str, _, state = StepString(str, state) 			resultRunes = []rune(c)

word_test.go

@@ -146,10 +146,10 @@ func TestWordCasesString(t *testing.T) {  // Benchmark the use of the word break function for byte slices. func BenchmarkWordFunctionBytes(b *testing.B) {-	str := []byte(benchmarkStr) 	for i := 0; i < b.N; i++ { 		var c []byte 		state := -1+		str := benchmarkBytes 		for len(str) > 0 { 			c, str, state = FirstWord(str, state) 			resultRunes = []rune(string(c))@@ -159,10 +159,10 @@ func BenchmarkWordFunctionBytes(b *testing.B) {  // Benchmark the use of the word break function for strings. func BenchmarkWordFunctionString(b *testing.B) {-	str := benchmarkStr 	for i := 0; i < b.N; i++ { 		var c string 		state := -1+		str := benchmarkStr 		for len(str) > 0 { 			c, str, state = FirstWordInString(str, state) 			resultRunes = []rune(c)

wordbreak_test.go

@@ -1,10 +1,10 @@-package uniseg- // Code generated via go generate from gen_breaktest.go. DO NOT EDIT. +package uniseg+ // wordBreakTestCases are Grapheme testcases taken from // https://www.unicode.org/Public/14.0.0/ucd/auxiliary/WordBreakTest.txt-// on September 10, 2022. See+// on July 17, 2023. See // https://www.unicode.org/license.html for the Unicode license agreement. var wordBreakTestCases = []testCase{ 	{original: "\u0001\u0001", expected: [][]rune{{0x0001}, {0x0001}}},                                                                                 // ÷ [0.2] <START OF HEADING> (Other) ÷ [999.0] <START OF HEADING> (Other) ÷ [0.3]

wordproperties.go

@@ -1,13 +1,13 @@-package uniseg- // Code generated via go generate from gen_properties.go. DO NOT EDIT. +package uniseg+ // workBreakCodePoints are taken from // https://www.unicode.org/Public/14.0.0/ucd/auxiliary/WordBreakProperty.txt // and // https://unicode.org/Public/14.0.0/ucd/emoji/emoji-data.txt // ("Extended_Pictographic" only)-// on September 10, 2022. See https://www.unicode.org/license.html for the Unicode+// on July 17, 2023. See https://www.unicode.org/license.html for the Unicode // license agreement. var workBreakCodePoints = [][3]int{ 	{0x000A, 0x000A, prLF},                     // Cc       <control-000A>

wordrules.go

@@ -97,7 +97,7 @@ var wbTransitions = map[[2]int][3]int{ 	{wbExtendNumLet, prALetter}:      {wbALetter, wbDontBreak, 132}, 	{wbExtendNumLet, prHebrewLetter}: {wbHebrewLetter, wbDontBreak, 132}, 	{wbExtendNumLet, prNumeric}:      {wbNumeric, wbDontBreak, 132},-	{wbExtendNumLet, prKatakana}:     {prKatakana, wbDontBreak, 132},+	{wbExtendNumLet, prKatakana}:     {wbKatakana, wbDontBreak, 132}, }  // transitionWordBreakState determines the new state of the word break parser