vt/coverage_test.go

@@ -16,7 +16,9 @@ package vt  import ( 	"testing"+	"unicode/utf8" +	"github.com/clipperhouse/uax29/v2/graphemes" 	"github.com/gdamore/tcell/v3/color" ) @@ -233,6 +235,10 @@ func TestShouldCheckGrapheme(t *testing.T) { 		{name: "vs16", prev: 'x', r: '\uFE0F', want: true}, 		{name: "supplementary vs", prev: 'x', r: '\U000E0100', want: true}, 		{name: "plain non-ascii", prev: 'x', r: 'π', want: false},+		// runes Go's unicode tables get wrong, and the segmenter does not+		{name: "zwnj", prev: 'a', r: '‌', want: true},+		{name: "emoji modifier", prev: 'a', r: '\U0001F3FB', want: true},+		{name: "unicode 16 mark", prev: 'a', r: '᫏', want: true}, 	}  	for _, tc := range cases {@@ -247,6 +253,40 @@ func TestShouldCheckGrapheme(t *testing.T) { 	} } +// TestGraphemeJoinerBlocks re-derives graphemeJoinerBlocks from the segmenter.+// shouldCheckGrapheme is only allowed to skip the segmenter for runes the+// segmenter would never join, so a stale table after a uax29 upgrade has to+// fail here rather than silently split clusters again.+func TestGraphemeJoinerBlocks(t *testing.T) {+	t.Parallel()++	buf := make([]byte, 0, 1+utf8.UTFMax)+	joiners, skipped := 0, 0+	for r := rune(utf8.RuneSelf); r <= utf8.MaxRune; r++ {+		if !utf8.ValidRune(r) {+			continue+		}+		buf = utf8.AppendRune(append(buf[:0], 'e'), r)+		iter := graphemes.FromBytes(buf)+		joins := iter.Next() && len(iter.Value()) == len(buf)+		check := shouldCheckGrapheme('e', r)+		if joins {+			joiners+++			if !check {+				t.Fatalf("U+%04X joins an ASCII base but graphemeJoinerBlocks skips it", r)+			}+		}+		if !check {+			skipped+++		}+	}+	// Guard against a table of all ones, which would pass the check above+	// while making the fast path pointless.+	if joiners == 0 || skipped < joiners {+		t.Fatalf("implausible table: %d joiners, %d runes skipped", joiners, skipped)+	}+}+ func TestPutRunePaths(t *testing.T) { 	newEm := func(width int) *emulator { 		em := NewEmulator(NewMockBackend(MockOptSize{X: Col(width), Y: 1}, MockOptColors(0))).(*emulator)

vt/emulate.go

@@ -24,7 +24,6 @@ import ( 	"strconv" 	"strings" 	"sync"-	"unicode" 	"unicode/utf8"  	"github.com/clipperhouse/uax29/v2/graphemes"@@ -1970,16 +1969,9 @@ func (em *emulator) putRune(r rune) { 				if em.graphemeIter.Next() && len(em.graphemeIter.Value()) == len(buf) { 					// we are adding to a cluster 					cluster := em.graphemeIter.Value()-					width := em.cells[lastIdx].W-					if w := textWidthOptions.Rune(r); w > width {-						width = w-					}-					if isRegionalIndicator(r) && width < 2 {-						width = 2-					}-					if r == '\uFE0F' && width < 2 {-						width = 2-					}+					// not the same as the widest rune in the cluster: a flag+					// is two width-1 runes but occupies two columns.+					width := textWidthOptions.Bytes(cluster) 					em.cells[lastIdx].C = em.clusterString(cluster) 					em.cells[lastIdx].W = width 					col := Col(lastIdx) % dim.X@@ -2052,29 +2044,15 @@ func (em *emulator) clusterString(cluster []byte) string { 	return em.clusterStrings.stringFor(cluster) } +// shouldCheckGrapheme reports whether r might extend a cluster ending in the+// ASCII byte prev.  It may say yes needlessly, but never no wrongly. func shouldCheckGrapheme(prev byte, r rune) bool { 	if r < utf8.RuneSelf {+		// an ASCII pair never clusters, except CR LF 		return prev == '\r' && r == '\n' 	}--	if unicode.Is(unicode.M, r) {-		return true-	}-	if r == '\u200d' {-		return true-	}-	if r >= 0xFE00 && r <= 0xFE0F {-		return true-	}-	if r >= 0xE0100 && r <= 0xE01EF {-		return true-	}--	return false-}--func isRegionalIndicator(r rune) bool {-	return r >= 0x1F1E6 && r <= 0x1F1FF+	b := uint32(r) >> 6+	return graphemeJoinerBlocks[b>>6]&(1<<(b&63)) != 0 }  // eraseCell erases a single cell at the given offset.

vt/graphemetable.go

new file mode 100644@@ -0,0 +1,41 @@+// Copyright 2026 The TCell Authors+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+//    http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package vt++// graphemeJoinerBlocks has one bit per 64 runes, set if any rune in that+// block can extend a grapheme cluster.  A clear bit means the segmenter+// cannot join anything in the block, so putRune may skip it.  Only 2619 of+// the 1113984 runes above U+007F can join, so nearly every bit is clear.+//+// The table is derived from the segmenter, not from Go's unicode tables,+// and TestGraphemeJoinerBlocks re-derives it to catch drift.+var graphemeJoinerBlocks = [272]uint64{+	0:   0xfffffffffbc43000,+	1:   0x0089ff15f0002007,+	2:   0x00a8000000000009,+	3:   0x0000000000000005,+	10:  0x00008ffd0e000000,+	15:  0x4100100000000000,+	16:  0x6c30090000002880,+	17:  0x387527b117cffbff,+	19:  0x0000000000020000,+	22:  0xe000180000000010,+	27:  0x0004000000000000,+	28:  0x3000000000000000,+	29:  0x0000070000000260,+	30:  0x0000002808880c15,+	31:  0x0000000000008000,+	224: 0x00000000000000f3,+}

vt/tests/mock_test.go

@@ -1300,6 +1300,38 @@ func TestGraphemeCluster(t *testing.T) { 	CheckContent(t, term, 0, 1, "A") } +// Characters that attach to an ASCII base must cluster with it even when+// Go's unicode tables do not classify them as marks. U+200C is a format+// character (Cf), U+1F3FB a modifier symbol (Sk), and U+1ACF is a combining+// mark that Go's tables, currently Unicode 15, still report as unassigned.+func TestGraphemeClusterAsciiBase(t *testing.T) {+	for _, tc := range []struct {+		name string+		text string+		next Col+	}{+		{"zwnj", "a\u200c", 1},+		{"emoji modifier", "a\U0001f3fb", 1},+		{"recent combining mark", "a\u1acf", 1},+	} {+		t.Run(tc.name, func(t *testing.T) {+			term := vt.NewMockTerm(vt.MockOptSize{X: 10, Y: 5})+			defer MustClose(t, term)+			MustStart(t, term)++			WriteF(t, term, "\x1b[?2027h")+			WriteF(t, term, "\x1b[H\x1b[J")+			WriteF(t, term, "%s", tc.text)++			CheckContent(t, term, 0, 0, tc.text)+			CheckPos(t, term, tc.next, 0)++			// the cell past the cluster must not have been written to+			CheckContent(t, term, tc.next, 0, "")+		})+	}+}+ func TestEraseAbove(t *testing.T) { 	term := vt.NewMockTerm(vt.MockOptSize{X: 10, Y: 5}) 	defer MustClose(t, term)