grapheme.go

@@ -187,8 +187,8 @@ func ReverseString(s string) string { const shiftGraphemePropState = 4  // FirstGraphemeCluster returns the first grapheme cluster found in the given-// byte slice according to the rules of Unicode Standard Annex #29, Grapheme-// Cluster Boundaries. This function can be called continuously to extract all+// byte slice according to the rules of [Unicode Standard Annex #29, Grapheme+// Cluster Boundaries]. This function can be called continuously to extract all // grapheme clusters from a byte slice, as illustrated in the example below. // // If you don't know the current state, for example when calling the function@@ -209,6 +209,8 @@ const shiftGraphemePropState = 4 // While slightly less convenient than using the Graphemes class, this function // has much better performance and makes no allocations. It lends itself well to // large byte slices.+//+// [Unicode Standard Annex #29, Grapheme Cluster Boundaries]: http://unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries func FirstGraphemeCluster(b []byte, state int) (cluster, rest []byte, width, newState int) { 	// An empty byte slice returns nothing. 	if len(b) == 0 {

line.go

@@ -4,7 +4,7 @@ import "unicode/utf8"  // FirstLineSegment returns the prefix of the given byte slice after which a // decision to break the string over to the next line can or must be made,-// according to the rules of Unicode Standard Annex #14. This is used to+// according to the rules of [Unicode Standard Annex #14]. This is used to // implement line breaking. // // Line breaking, also known as word wrapping, is the process of breaking a@@ -35,7 +35,7 @@ import "unicode/utf8" // // Given an empty byte slice "b", the function returns nil values. //-// Note that in accordance with UAX #14 LB3, the final segment will end with+// Note that in accordance with [UAX #14 LB3], the final segment will end with // "mustBreak" set to true. You can choose to ignore this by checking if the // length of the "rest" slice is 0 and calling [HasTrailingLineBreak] or // [HasTrailingLineBreakInString] on the last rune.@@ -43,6 +43,9 @@ import "unicode/utf8" // Note also that this algorithm may break within grapheme clusters. This is // addressed in Section 8.2 Example 6 of UAX #14. To avoid this, you can use // the [Step] function instead.+//+// [Unicode Standard Annex #14]: https://www.unicode.org/reports/tr14/+// [UAX #14 LB3]: https://www.unicode.org/reports/tr14/#Algorithm func FirstLineSegment(b []byte, state int) (segment, rest []byte, mustBreak bool, newState int) { 	// An empty byte slice returns nothing. 	if len(b) == 0 {

sentence.go

@@ -3,7 +3,7 @@ package uniseg import "unicode/utf8"  // FirstSentence returns the first sentence found in the given byte slice-// according to the rules of Unicode Standard Annex #29, Sentence Boundaries.+// according to the rules of [Unicode Standard Annex #29, Sentence Boundaries]. // This function can be called continuously to extract all sentences from a byte // slice, as illustrated in the example below. //@@ -17,6 +17,8 @@ import "unicode/utf8" // slice is the sub-slice of the input slice containing the identified sentence. // // Given an empty byte slice "b", the function returns nil values.+//+// [Unicode Standard Annex #29, Sentence Boundaries]: http://unicode.org/reports/tr29/#Sentence_Boundaries func FirstSentence(b []byte, state int) (sentence, rest []byte, newState int) { 	// An empty byte slice returns nothing. 	if len(b) == 0 {

step.go

@@ -83,10 +83,12 @@ const ( // has much better performance and makes no allocations. It lends itself well to // large byte slices. //-// Note that in accordance with UAX #14 LB3, the final segment will end with+// Note that in accordance with [UAX #14 LB3], the final segment will end with // a mandatory line break (boundaries&MaskLine == LineMustBreak). You can choose // to ignore this by checking if the length of the "rest" slice is 0 and calling // [HasTrailingLineBreak] or [HasTrailingLineBreakInString] on the last rune.+//+// [UAX #14 LB3]: https://www.unicode.org/reports/tr14/#Algorithm func Step(b []byte, state int) (cluster, rest []byte, boundaries int, newState int) { 	// An empty byte slice returns nothing. 	if len(b) == 0 {

step_test.go

@@ -1,6 +1,8 @@ package uniseg -import "testing"+import (+	"testing"+)  // Test official Grapheme Cluster Unicode test cases for grapheme clusters using // the [Step] function.@@ -455,3 +457,48 @@ func BenchmarkStepString(b *testing.B) { 		} 	} }++// Fuzz the StepString function.+func FuzzStepString(f *testing.F) {+	for _, tc := range graphemeBreakTestCases {+		f.Add(tc.original)+	}+	f.Fuzz(func(t *testing.T, orig string) {+		var (+			c          string+			b          []byte+			boundaries int+		)+		str := orig+		state := -1+		for len(str) > 0 {+			c, str, boundaries, state = StepString(str, state)+			b = append(b, []byte(c)...)+		}++		// Check if the constructed string is the same as the original.+		if string(b) != orig {+			t.Errorf("Fuzzing failed: %q != %q", string(b), orig)+		}++		// For all other checks, we need to have a non-empty string.+		if orig == "" {+			return+		}++		// Check end boundaries.+		if boundaries&MaskWord == 0 {+			t.Errorf("String %q does not end on a word boundary (final boundary = %x)", orig, state)+		}+		if boundaries&MaskSentence == 0 {+			t.Errorf("String %q does not end on a sentence boundary (final boundary = %x)", orig, state)+		}+		if boundaries&MaskLine != LineMustBreak {+			t.Errorf("String %q does not end with a mandatory line break (final boundary = %x)", orig, state)+		}++		// Note: If you have ideas for more useful checks we could add here,+		// please submit them here:+		// https://github.com/rivo/uniseg/issues+	})+}

word.go

@@ -3,7 +3,7 @@ package uniseg import "unicode/utf8"  // FirstWord returns the first word found in the given byte slice according to-// the rules of Unicode Standard Annex #29, Word Boundaries. This function can+// the rules of [Unicode Standard Annex #29, Word Boundaries]. This function can // be called continuously to extract all words from a byte slice, as illustrated // in the example below. //@@ -17,6 +17,8 @@ import "unicode/utf8" // the sub-slice of the input slice containing the identified word. // // Given an empty byte slice "b", the function returns nil values.+//+// [Unicode Standard Annex #29, Word Boundaries]: http://unicode.org/reports/tr29/#Word_Boundaries func FirstWord(b []byte, state int) (word, rest []byte, newState int) { 	// An empty byte slice returns nothing. 	if len(b) == 0 {