Keep word motion working in REPL output

CHANGELOG.md

@@ -34,6 +34,7 @@
 ### Bugs fixed
 
 - [#4182](https://github.com/clojure-emacs/cider/issues/4182): Stop the server-output subscription made on connect from overwriting the `cider-eval-register` register (`e` by default) with an empty string.
+- [#4154](https://github.com/clojure-emacs/cider/issues/4154): Keep word motion and `isearch-yank-word-or-char` working in REPL output: only the delimiters in output get the sexp-neutralizing punctuation syntax now, not every character, so `M-f`/`M-b` no longer jump over a whole output block.
 - [#4180](https://github.com/clojure-emacs/cider/pull/4180): Accept a `:port` given as a number, not just a string, in `cider-connect` and friends: the two spellings are now stored the same way, so a connection made with one is recognized when looked up with the other.
 - [#4172](https://github.com/clojure-emacs/cider/pull/4172): Stop treating the text of a line comment as code: with point in or after a comment, `cider-eval-last-sexp` and friends now target the form preceding the comment, and the in-place macroexpansion commands no longer overwrite the comment with an expansion.
 - [#4158](https://github.com/clojure-emacs/cider/pull/4158): Detect running `lein trampoline` REPLs in `cider-connect`'s port suggestions (the trampolined JVM has no "leiningen" marker for the process scan to find), and don't miss REPLs running without a controlling terminal.

lisp/cider-repl.el

@@ -918,6 +918,24 @@ Clear the part where `cider-repl-buffer-size-limit' is exceeded."
   (when (> (buffer-size) cider-repl-buffer-size-limit)
     (cider-repl-trim-top-of-buffer buffer)))
 
+(defvar cider-repl-mode-syntax-table
+  (copy-syntax-table clojure-mode-syntax-table))
+
+(defvar cider-repl--output-syntax-table
+  (let ((table (make-syntax-table cider-repl-mode-syntax-table)))
+    ;; Turn every delimiter - parens, string quotes, comment starters,
+    ;; escapes and their generic variants - into punctuation, leaving words,
+    ;; symbols and whitespace as they are.
+    (map-char-table (lambda (range syntax)
+                      (when (memq (syntax-class syntax) '(4 5 7 8 9 10 11 14 15))
+                        (modify-syntax-entry range "." table)))
+                    cider-repl-mode-syntax-table)
+    table)
+  "Syntax table for REPL output, with every delimiter demoted to punctuation.
+Attached to output via the `syntax-table' text property (see #3102), so
+unbalanced parens or quotes in it can't confuse the sexp scanner, while word
+motion and isearch keep working on the rest (see #4154).")
+
 (defun cider-repl--emit-output (buffer string face)
   "Using BUFFER, emit STRING as output font-locked using FACE.
 Before inserting, run `cider-repl-preoutput-hook' on STRING."
@@ -929,13 +947,14 @@ Before inserting, run `cider-repl-preoutput-hook' on STRING."
                                  'font-lock-face face
                                  'rear-nonsticky '(font-lock-face)))
         (setq string (cider-run-chained-hook 'cider-repl-preoutput-hook string))
-        ;; #3102: give output punctuation syntax so unbalanced parens, brackets
-        ;; or quotes in it don't break sexp navigation or paredit in the REPL.
-        ;; Applied after the preoutput hooks (e.g. ANSI coloring) so it survives
-        ;; on the final string.  Relies on `parse-sexp-lookup-properties', set
-        ;; in `cider-repl-mode'.
+        ;; #3102: give output a syntax table where delimiters are punctuation,
+        ;; so unbalanced parens, brackets or quotes in it don't break sexp
+        ;; navigation or paredit in the REPL.  Applied after the preoutput hooks
+        ;; (e.g. ANSI coloring) so it survives on the final string.  Relies on
+        ;; `parse-sexp-lookup-properties', set in `cider-repl-mode'.
         (add-text-properties 0 (length string)
-                             '(syntax-table (1) rear-nonsticky (font-lock-face syntax-table))
+                             `(syntax-table ,cider-repl--output-syntax-table
+                               rear-nonsticky (font-lock-face syntax-table))
                              string)
         (insert-before-markers string))
       (when (and (= (point) cider-repl-prompt-start-mark)
@@ -2164,9 +2183,6 @@ the history file is rewritten if `cider-repl-history-file' is set."
 (defvar cider-repl-mode-hook nil
   "Hook executed when entering `cider-repl-mode'.")
 
-(defvar cider-repl-mode-syntax-table
-  (copy-syntax-table clojure-mode-syntax-table))
-
 (defconst cider-repl--prettify-symbols-alist
   '(("fn" . ?λ))
   "Alist used to seed `prettify-symbols-alist' in the REPL buffer.")
@@ -2356,6 +2372,9 @@ the history file is rewritten if `cider-repl-history-file' is set."
   ;; #3102: honor the `syntax-table' text property that `cider-repl--emit-output'
   ;; puts on output, so unbalanced parens in output don't break sexp commands.
   (setq-local parse-sexp-lookup-properties t)
+  ;; ...but don't let it travel with text yanked from output into the input,
+  ;; where the parens have to count again for `cider-repl--input-complete-p'.
+  (setq-local yank-excluded-properties (cons 'syntax-table yank-excluded-properties))
   (cider-eldoc-setup)
   ;; At the REPL, we define beginning-of-defun and end-of-defun to be
   ;; the start of the previous prompt or next prompt respectively.

test/cider-repl-tests.el

@@ -245,10 +245,47 @@
       (cider-repl-reset-markers)
       (cider-repl--emit-output (current-buffer) ")))))" 'cider-repl-stdout-face)
       ;; the output parens carry punctuation syntax, not close-paren syntax
-      (expect (get-text-property (point-min) 'syntax-table)
-              :to-equal (string-to-syntax "."))
+      (expect (syntax-after (point-min)) :to-equal (string-to-syntax "."))
       ;; so scanning backwards across them no longer raises a scan-error
-      (expect (ignore-errors (scan-sexps (point-max) -1) t) :to-be-truthy))))
+      (expect (ignore-errors (scan-sexps (point-max) -1) t) :to-be-truthy)))
+
+  (it "neutralizes non-ASCII delimiters too"
+    (with-temp-buffer
+      (clojure-mode)
+      (setq-local parse-sexp-lookup-properties t)
+      (cider-repl-reset-markers)
+      (cider-repl--emit-output (current-buffer) "oops）「x" 'cider-repl-stdout-face)
+      (expect (syntax-after 5) :to-equal (string-to-syntax "."))
+      (expect (syntax-after 6) :to-equal (string-to-syntax "."))))
+
+  (it "neutralizes only the delimiters, so word motion still works in output (#4154)"
+    (with-temp-buffer
+      (clojure-mode)
+      (setq-local parse-sexp-lookup-properties t)
+      (cider-repl-reset-markers)
+      (cider-repl--emit-output (current-buffer) "FAIL in (foo-test) (core_test.clj:1)"
+                               'cider-repl-stdout-face)
+      ;; the parens are punctuation, the words keep their word syntax
+      (expect (syntax-after 9) :to-equal (string-to-syntax "."))
+      (expect (syntax-after (point-min)) :to-equal (string-to-syntax "w"))
+      (goto-char (point-min))
+      (forward-word)
+      (expect (point) :to-equal 5)      ; after "FAIL"
+      (forward-word)
+      (expect (point) :to-equal 8)))    ; after "in", not the end of the output
+
+  (it "doesn't carry the neutralized syntax over to text yanked from output"
+    (with-temp-buffer
+      (cider-repl-mode)
+      (cider-repl-reset-markers)
+      (cider-repl--emit-output (current-buffer) "(foo bar" 'cider-repl-stdout-face)
+      (kill-new (buffer-substring (point-min) (point-max)))
+      (goto-char (point-max))
+      (let ((yank-start (point)))
+        (yank)
+        ;; the yanked paren counts again
+        (expect (get-text-property yank-start 'syntax-table) :to-be nil)
+        (expect (syntax-class (syntax-after yank-start)) :to-equal 4)))))
 
 (describe "cider-repl-emit-result with streamed (chunked) values"
   (it "font-locks a value streamed in several chunks as one whole form"