diff --git a/packages/tui/src/components/editor.ts b/packages/tui/src/components/editor.ts index 0b30ad0a..fad33c64 100644 --- a/packages/tui/src/components/editor.ts +++ b/packages/tui/src/components/editor.ts @@ -4,10 +4,11 @@ import { decodePrintableKey, matchesKey } from "../keys.ts"; import { KillRing } from "../kill-ring.ts"; import { type Component, CURSOR_MARKER, type Focusable, type TUI } from "../tui.ts"; import { UndoStack } from "../undo-stack.ts"; -import { getSegmenter, isPunctuationChar, isWhitespaceChar, truncateToWidth, visibleWidth } from "../utils.ts"; +import { getGraphemeSegmenter, getWordSegmenter, isWhitespaceChar, truncateToWidth, visibleWidth } from "../utils.ts"; import { SelectList, type SelectListLayoutOptions, type SelectListTheme } from "./select-list.ts"; -const baseSegmenter = getSegmenter(); +const graphemeSegmenter = getGraphemeSegmenter(); +const wordSegmenter = getWordSegmenter(); /** Regex matching paste markers like `[paste #1 +123 lines]` or `[paste #2 1234 chars]`. */ const PASTE_MARKER_REGEX = /\[paste #(\d+)( (\+\d+ lines|\d+ chars))?\]/g; @@ -27,7 +28,11 @@ function isPasteMarker(segment: string): boolean { * * Only markers whose numeric ID exists in `validIds` are merged. */ -function segmentWithMarkers(text: string, validIds: Set): Iterable { +function segmentWithMarkers( + text: string, + baseSegmenter: Intl.Segmenter, + validIds: Set, +): Iterable { // Fast path: no paste markers in the text or no valid IDs. if (validIds.size === 0 || !text.includes("[paste #")) { return baseSegmenter.segment(text); @@ -109,7 +114,7 @@ export function wordWrapLine(line: string, maxWidth: number, preSegmented?: Intl } const chunks: TextChunk[] = []; - const segments = preSegmented ?? [...baseSegmenter.segment(line)]; + const segments = preSegmented ?? [...graphemeSegmenter.segment(line)]; let currentWidth = 0; let chunkStart = 0; @@ -301,8 +306,8 @@ export class Editor implements Component, Focusable { } /** Segment text with paste-marker awareness, only merging markers with valid IDs. */ - private segment(text: string): Iterable { - return segmentWithMarkers(text, this.validPasteIds()); + private segment(text: string, mode: "word" | "grapheme"): Iterable { + return segmentWithMarkers(text, mode === "word" ? wordSegmenter : graphemeSegmenter, this.validPasteIds()); } getPaddingX(): number { @@ -482,7 +487,7 @@ export class Editor implements Component, Focusable { if (after.length > 0) { // Cursor is on a character (grapheme) - replace it with highlighted version // Get the first grapheme from 'after' - const afterGraphemes = [...this.segment(after)]; + const afterGraphemes = [...this.segment(after, "grapheme")]; const firstGrapheme = afterGraphemes[0]?.segment || ""; const restAfter = after.slice(firstGrapheme.length); const cursor = `\x1b[7m${firstGrapheme}\x1b[0m`; @@ -855,7 +860,7 @@ export class Editor implements Component, Focusable { } } else { // Line needs wrapping - use word-aware wrapping - const chunks = wordWrapLine(line, contentWidth, [...this.segment(line)]); + const chunks = wordWrapLine(line, contentWidth, [...this.segment(line, "grapheme")]); for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) { const chunk = chunks[chunkIndex]; @@ -1213,7 +1218,7 @@ export class Editor implements Component, Focusable { const beforeCursor = line.slice(0, this.state.cursorCol); // Find the last grapheme in the text before cursor - const graphemes = [...this.segment(beforeCursor)]; + const graphemes = [...this.segment(beforeCursor, "grapheme")]; const lastGrapheme = graphemes[graphemes.length - 1]; const graphemeLength = lastGrapheme ? lastGrapheme.segment.length : 1; @@ -1314,7 +1319,7 @@ export class Editor implements Component, Focusable { // Snap cursor to atomic segment boundary (e.g. paste markers) // so the cursor never lands in the middle of a multi-grapheme unit. // Single-grapheme segments don't need snapping. - const segments = [...this.segment(logicalLine)]; + const segments = [...this.segment(logicalLine, "grapheme")]; for (const seg of segments) { if (seg.index > this.state.cursorCol) break; if (seg.segment.length <= 1) continue; @@ -1585,7 +1590,7 @@ export class Editor implements Component, Focusable { const afterCursor = currentLine.slice(this.state.cursorCol); // Find the first grapheme at cursor - const graphemes = [...this.segment(afterCursor)]; + const graphemes = [...this.segment(afterCursor, "grapheme")]; const firstGrapheme = graphemes[0]; const graphemeLength = firstGrapheme ? firstGrapheme.segment.length : 1; @@ -1642,7 +1647,7 @@ export class Editor implements Component, Focusable { visualLines.push({ logicalLine: i, startCol: 0, length: line.length }); } else { // Line needs wrapping - use word-aware wrapping - const chunks = wordWrapLine(line, width, [...this.segment(line)]); + const chunks = wordWrapLine(line, width, [...this.segment(line, "grapheme")]); for (const chunk of chunks) { visualLines.push({ logicalLine: i, @@ -1707,7 +1712,7 @@ export class Editor implements Component, Focusable { // Moving right - move by one grapheme (handles emojis, combining characters, etc.) if (this.state.cursorCol < currentLine.length) { const afterCursor = currentLine.slice(this.state.cursorCol); - const graphemes = [...this.segment(afterCursor)]; + const graphemes = [...this.segment(afterCursor, "grapheme")]; const firstGrapheme = graphemes[0]; this.setCursorCol(this.state.cursorCol + (firstGrapheme ? firstGrapheme.segment.length : 1)); } else if (this.state.cursorLine < this.state.lines.length - 1) { @@ -1725,7 +1730,7 @@ export class Editor implements Component, Focusable { // Moving left - move by one grapheme (handles emojis, combining characters, etc.) if (this.state.cursorCol > 0) { const beforeCursor = currentLine.slice(0, this.state.cursorCol); - const graphemes = [...this.segment(beforeCursor)]; + const graphemes = [...this.segment(beforeCursor, "grapheme")]; const lastGrapheme = graphemes[graphemes.length - 1]; this.setCursorCol(this.state.cursorCol - (lastGrapheme ? lastGrapheme.segment.length : 1)); } else if (this.state.cursorLine > 0) { @@ -1769,41 +1774,32 @@ export class Editor implements Component, Focusable { } const textBeforeCursor = currentLine.slice(0, this.state.cursorCol); - const graphemes = [...this.segment(textBeforeCursor)]; + const segments = [...this.segment(textBeforeCursor, "word")]; let newCol = this.state.cursorCol; // Skip trailing whitespace while ( - graphemes.length > 0 && - !isPasteMarker(graphemes[graphemes.length - 1]?.segment || "") && - isWhitespaceChar(graphemes[graphemes.length - 1]?.segment || "") + segments.length > 0 && + !isPasteMarker(segments[segments.length - 1]?.segment || "") && + isWhitespaceChar(segments[segments.length - 1]?.segment || "") ) { - newCol -= graphemes.pop()?.segment.length || 0; + newCol -= segments.pop()?.segment.length || 0; } - if (graphemes.length > 0) { - const lastGrapheme = graphemes[graphemes.length - 1]?.segment || ""; - if (isPasteMarker(lastGrapheme)) { - // Paste marker is a single atomic word - newCol -= graphemes.pop()?.segment.length || 0; - } else if (isPunctuationChar(lastGrapheme)) { - // Skip punctuation run - while ( - graphemes.length > 0 && - isPunctuationChar(graphemes[graphemes.length - 1]?.segment || "") && - !isPasteMarker(graphemes[graphemes.length - 1]?.segment || "") - ) { - newCol -= graphemes.pop()?.segment.length || 0; - } + if (segments.length > 0) { + const last = segments[segments.length - 1]!; + if (isPasteMarker(last.segment) || last.isWordLike) { + // Skip one word-like segment (or paste marker) + newCol -= segments.pop()?.segment.length || 0; } else { - // Skip word run + // Skip non-word non-whitespace run (punctuation) while ( - graphemes.length > 0 && - !isWhitespaceChar(graphemes[graphemes.length - 1]?.segment || "") && - !isPunctuationChar(graphemes[graphemes.length - 1]?.segment || "") && - !isPasteMarker(graphemes[graphemes.length - 1]?.segment || "") + segments.length > 0 && + !isPasteMarker(segments[segments.length - 1]?.segment || "") && + !segments[segments.length - 1]?.isWordLike && + !isWhitespaceChar(segments[segments.length - 1]?.segment || "") ) { - newCol -= graphemes.pop()?.segment.length || 0; + newCol -= segments.pop()?.segment.length || 0; } } } @@ -1996,7 +1992,7 @@ export class Editor implements Component, Focusable { } const textAfterCursor = currentLine.slice(this.state.cursorCol); - const segments = this.segment(textAfterCursor); + const segments = this.segment(textAfterCursor, "word"); const iterator = segments[Symbol.iterator](); let next = iterator.next(); let newCol = this.state.cursorCol; @@ -2008,23 +2004,16 @@ export class Editor implements Component, Focusable { } if (!next.done) { - const firstGrapheme = next.value.segment; - if (isPasteMarker(firstGrapheme)) { - // Paste marker is a single atomic word - newCol += firstGrapheme.length; - } else if (isPunctuationChar(firstGrapheme)) { - // Skip punctuation run - while (!next.done && isPunctuationChar(next.value.segment) && !isPasteMarker(next.value.segment)) { - newCol += next.value.segment.length; - next = iterator.next(); - } + if (isPasteMarker(next.value.segment) || next.value.isWordLike) { + // Skip one word-like segment (or paste marker) + newCol += next.value.segment.length; } else { - // Skip word run + // Skip non-word non-whitespace run (punctuation) while ( !next.done && - !isWhitespaceChar(next.value.segment) && - !isPunctuationChar(next.value.segment) && - !isPasteMarker(next.value.segment) + !isPasteMarker(next.value.segment) && + !next.value.isWordLike && + !isWhitespaceChar(next.value.segment) ) { newCol += next.value.segment.length; next = iterator.next(); diff --git a/packages/tui/src/components/input.ts b/packages/tui/src/components/input.ts index 71f2363b..2f5a3eb6 100644 --- a/packages/tui/src/components/input.ts +++ b/packages/tui/src/components/input.ts @@ -3,9 +3,9 @@ import { decodeKittyPrintable } from "../keys.ts"; import { KillRing } from "../kill-ring.ts"; import { type Component, CURSOR_MARKER, type Focusable } from "../tui.ts"; import { UndoStack } from "../undo-stack.ts"; -import { getSegmenter, isPunctuationChar, isWhitespaceChar, sliceByColumn, visibleWidth } from "../utils.ts"; +import { getGraphemeSegmenter, isPunctuationChar, isWhitespaceChar, sliceByColumn, visibleWidth } from "../utils.ts"; -const segmenter = getSegmenter(); +const segmenter = getGraphemeSegmenter(); interface InputState { value: string; diff --git a/packages/tui/src/utils.ts b/packages/tui/src/utils.ts index a6bcc8a5..b3d336a2 100644 --- a/packages/tui/src/utils.ts +++ b/packages/tui/src/utils.ts @@ -1,13 +1,21 @@ import { eastAsianWidth } from "get-east-asian-width"; -// Grapheme segmenter (shared instance) -const segmenter = new Intl.Segmenter(undefined, { granularity: "grapheme" }); +// segmenters (shared instance) +const graphemeSegmenter = new Intl.Segmenter(undefined, { granularity: "grapheme" }); +const wordSegmenter = new Intl.Segmenter(undefined, { granularity: "word" }); /** * Get the shared grapheme segmenter instance. */ -export function getSegmenter(): Intl.Segmenter { - return segmenter; +export function getGraphemeSegmenter(): Intl.Segmenter { + return graphemeSegmenter; +} + +/** + * Get the shared word segmenter instance. + */ +export function getWordSegmenter(): Intl.Segmenter { + return wordSegmenter; } /** @@ -62,7 +70,7 @@ function truncateFragmentToWidth(text: string, maxWidth: number): { text: string if (!hasAnsi && !hasTabs) { let result = ""; let width = 0; - for (const { segment } of segmenter.segment(text)) { + for (const { segment } of graphemeSegmenter.segment(text)) { const w = graphemeWidth(segment); if (width + w > maxWidth) { break; @@ -109,7 +117,7 @@ function truncateFragmentToWidth(text: string, maxWidth: number): { text: string end++; } - for (const { segment } of segmenter.segment(text.slice(i, end))) { + for (const { segment } of graphemeSegmenter.segment(text.slice(i, end))) { const w = graphemeWidth(segment); if (width + w > maxWidth) { return { text: result, width }; @@ -239,7 +247,7 @@ export function visibleWidth(str: string): number { // Calculate width let width = 0; - for (const { segment } of segmenter.segment(clean)) { + for (const { segment } of graphemeSegmenter.segment(clean)) { width += graphemeWidth(segment); } @@ -790,7 +798,7 @@ function breakLongWord(word: string, width: number, tracker: AnsiCodeTracker): s } // Segment this non-ANSI portion into graphemes const textPortion = word.slice(i, end); - for (const seg of segmenter.segment(textPortion)) { + for (const seg of graphemeSegmenter.segment(textPortion)) { segments.push({ type: "grapheme", value: seg.segment }); } i = end; @@ -912,7 +920,7 @@ export function truncateToWidth( const hasTabs = text.includes("\t"); if (!hasAnsi && !hasTabs) { - for (const { segment } of segmenter.segment(text)) { + for (const { segment } of graphemeSegmenter.segment(text)) { const width = graphemeWidth(segment); if (keepContiguousPrefix && keptWidth + width <= targetWidth) { result += segment; @@ -967,7 +975,7 @@ export function truncateToWidth( end++; } - for (const { segment } of segmenter.segment(text.slice(i, end))) { + for (const { segment } of graphemeSegmenter.segment(text.slice(i, end))) { const width = graphemeWidth(segment); if (keepContiguousPrefix && keptWidth + width <= targetWidth) { if (pendingAnsi) { @@ -1037,7 +1045,7 @@ export function sliceWithWidth( let textEnd = i; while (textEnd < line.length && !extractAnsiCode(line, textEnd)) textEnd++; - for (const { segment } of segmenter.segment(line.slice(i, textEnd))) { + for (const { segment } of graphemeSegmenter.segment(line.slice(i, textEnd))) { const w = graphemeWidth(segment); const inRange = currentCol >= startCol && currentCol < endCol; const fits = !strict || currentCol + w <= endCol; @@ -1105,7 +1113,7 @@ export function extractSegments( let textEnd = i; while (textEnd < line.length && !extractAnsiCode(line, textEnd)) textEnd++; - for (const { segment } of segmenter.segment(line.slice(i, textEnd))) { + for (const { segment } of graphemeSegmenter.segment(line.slice(i, textEnd))) { const w = graphemeWidth(segment); if (currentCol < beforeEnd) { diff --git a/packages/tui/test/editor.test.ts b/packages/tui/test/editor.test.ts index e7e74ec3..6dc0540c 100644 --- a/packages/tui/test/editor.test.ts +++ b/packages/tui/test/editor.test.ts @@ -591,6 +591,82 @@ describe("Editor component", () => { editor.handleInput("\x1b[1;5C"); // Ctrl+Right assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 6 }); // after 'foo' }); + + it("stops at fullwidth Chinese punctuation (issue #4972)", () => { + const editor = new Editor(createTestTUI(), defaultEditorTheme); + + // 你好，世界 = 你好(0-2) ，(2-3) 世界(3-5) + editor.setText("你好，世界"); + // Cursor at end (col 5) + + // Move left over 世界 + editor.handleInput("\x1b[1;5D"); // Ctrl+Left + assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 3 }); // after ， + + // Move left over ， + editor.handleInput("\x1b[1;5D"); // Ctrl+Left + assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 2 }); // after 你好 + + // Move left over 你好 + editor.handleInput("\x1b[1;5D"); // Ctrl+Left + assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 0 }); // start + + // Move right over 你好 + editor.handleInput("\x1b[1;5C"); // Ctrl+Right + assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 2 }); // after 你好 + + // Move right over ， + editor.handleInput("\x1b[1;5C"); // Ctrl+Right + assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 3 }); // after ， + + // Move right over 世界 + editor.handleInput("\x1b[1;5C"); // Ctrl+Right + assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 5 }); // end + }); + + it("handles mixed CJK and ASCII word movement", () => { + const editor = new Editor(createTestTUI(), defaultEditorTheme); + + // "hello你好，world世界" = hello(0-5) 你好(5-7) ，(7-8) world(8-13) 世界(13-15) + editor.setText("hello你好，world世界"); + // Cursor at end (col 15) + + // Move left over 世界 + editor.handleInput("\x1b[1;5D"); // Ctrl+Left + assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 13 }); // after 'world' + + // Move left over world + editor.handleInput("\x1b[1;5D"); // Ctrl+Left + assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 8 }); // after ， + + // Move left over ， + editor.handleInput("\x1b[1;5D"); // Ctrl+Left + assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 7 }); // after 你好 + + // Move left over 你好 + editor.handleInput("\x1b[1;5D"); // Ctrl+Left + assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 5 }); // after 'hello' + + // Move left over hello + editor.handleInput("\x1b[1;5D"); // Ctrl+Left + assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 0 }); // start + + // Forward from start + editor.handleInput("\x1b[1;5C"); // Ctrl+Right + assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 5 }); // after 'hello' + + editor.handleInput("\x1b[1;5C"); // Ctrl+Right + assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 7 }); // after 你好 + + editor.handleInput("\x1b[1;5C"); // Ctrl+Right + assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 8 }); // after ， + + editor.handleInput("\x1b[1;5C"); // Ctrl+Right + assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 13 }); // after 'world' + + editor.handleInput("\x1b[1;5C"); // Ctrl+Right + assert.deepStrictEqual(editor.getCursor(), { line: 0, col: 15 }); // end + }); }); describe("Grapheme-aware text wrapping", () => {