Overhaul REPL input: token completion and whole-form reading

README.md

@@ -8,9 +8,11 @@ and is spoken by a growing family of servers and editor clients
 ([CIDER](https://cider.mx), [neat](https://github.com/nrepl/neat),
 [conjure](https://github.com/Olical/conjure) and friends). mezcaml is the
 OCaml counterpart: a small client library plus a command-line tool, in the
-spirit of neat's "do the protocol, skip the magic" philosophy. It pairs
-nicely with utop's (experimental, for now) nREPL server mode, but it should
-work against any nREPL server, whatever the language behind it.
+spirit of neat's "do the protocol, skip the magic" philosophy. It works
+against any nREPL server, whatever the language on the other end - the
+Clojure reference implementation, the BEAM servers
+([dialtone](https://github.com/nrepl/nrepl-beam) for Erlang, repartee for
+Elixir), and anything else that speaks the protocol.
 
 ## Status
 
@@ -27,22 +29,40 @@ dune install
 
 ## Command-line usage
 
-Start an nREPL server somewhere. For OCaml that would be:
+Start an nREPL server somewhere. The Clojure reference server, for
+instance:
 
 ```
-utop -nrepl 7888
+clj -Sdeps '{:deps {nrepl/nrepl {:mvn/version "1.7.0"}}}' -M -m nrepl.cmdline --port 7888
 ```
 
 Then:
 
 ```
 $ mezcaml -p 7888
-mezcaml, a minimal nREPL client. Ctrl-D to quit.
-Connected to 127.0.0.1:7888 (nrepl 1.0, ocaml 5.3.0, utop 2.16.0)
-mezcaml> 1 + 2
-- : int = 3
-mezcaml> let square x = x * x
-val square : int -> int = <fun>
+mezcaml 0.1.0, a minimal nREPL client. Ctrl-D to quit.
+Connected to 127.0.0.1:7888 (clojure 1.12.5, java 21.0.5, nrepl 1.7.0)
+mezcaml> (+ 1 2)
+3
+mezcaml> (defn square [x]
+    ...>   (* x x))
+#'user/square
+mezcaml> (square 7)
+49
+```
+
+The language on the other end makes no difference. Here's the same
+client talking to [dialtone](https://github.com/nrepl/nrepl-beam), the
+Erlang server (started with `bin/dialtone --port 7888`):
+
+```
+$ mezcaml -p 7888
+mezcaml 0.1.0, a minimal nREPL client. Ctrl-D to quit.
+Connected to 127.0.0.1:7888 (dialtone 0.1.0, erlang 29, nrepl 1.0.0)
+mezcaml> Square = fun(X) -> X * X end.
+#Fun<erl_eval.42.130099583>
+mezcaml> Square(7).
+49
 ```
 
 With no `-p`, mezcaml walks up from the current directory looking for the
@@ -51,21 +71,46 @@ does the right thing from inside a project.
 
 The REPL has line editing and persistent history (via linenoise; history
 lives in `$XDG_STATE_HOME/mezcaml/history`), and Tab completion powered by
-the server's `completions` op. Evaluation results are shown in green and
+the server's `completions` op. It reads whole forms, not lines: the
+`...>` prompt keeps going until brackets balance (or, on an Erlang
+server, until the closing `.`). Evaluation results are shown in green and
 errors in red when stdout is a terminal; set `NO_COLOR` to turn that off.
-Ctrl-C cancels the current line, Ctrl-D quits.
+
+### Key bindings
+
+The usual emacs-flavored readline bindings work:
+
+| Keys | Action |
+|---|---|
+| Ctrl-A / Ctrl-E | start / end of line |
+| Ctrl-B / Ctrl-F | move by character |
+| Alt-B / Alt-F, Ctrl-Left / Ctrl-Right | move by word |
+| Ctrl-W, Alt-Backspace | delete the word before the cursor |
+| Alt-D | delete the word after the cursor |
+| Ctrl-K | delete to the end of the line |
+| Ctrl-U | delete the whole line |
+| Ctrl-T | transpose characters |
+| Up / Down, Ctrl-P / Ctrl-N | walk the history |
+| Ctrl-R | incremental reverse history search; Ctrl-R again for older matches, Enter keeps the match, Ctrl-G puts your line back |
+| Ctrl-L | clear the screen |
+| Tab | completion, powered by the server |
+| Ctrl-C | cancel the input being edited |
+| Ctrl-D | delete the character under the cursor; on an empty line, quit |
+
+No kill ring and no undo - that's where linenoise draws the line, and
+mezcaml with it.
 
 One-shot evaluation (exit code reflects success, handy for scripts):
 
 ```
-$ mezcaml -e '21 * 2'
-- : int = 42
+$ mezcaml -e '(* 21 2)'
+42
 ```
 
 Send a whole file:
 
 ```
-$ mezcaml --load scratch.ml
+$ mezcaml --load scratch.clj
 ```
 
 For scripting there's also `--timeout SECONDS` (bounds connecting and each
@@ -79,7 +124,7 @@ and `--version`.
 let () =
   let conn = Mezcaml.Client.connect ~host:"127.0.0.1" ~port:7888 () in
   let _session = Mezcaml.Client.clone conn in
-  let responses = Mezcaml.Client.eval conn "1 + 2" in
+  let responses = Mezcaml.Client.eval conn "(+ 1 2)" in
   List.iter
     (fun msg ->
        match Mezcaml.Bencode.get_string msg "value" with
@@ -104,8 +149,10 @@ By design, at least for now:
 - Synchronous, one request in flight at a time. Fine for a CLI and simple
   tooling; an editor integration would want an async layer on top.
 - No TLS, no EDN transport, only the default bencode one.
-- Input is evaluated line by line; multi-line definitions have to go
-  through `--load` for now.
+- Whole-form reading is a heuristic: balanced brackets and strings for
+  Lisp-family servers, the terminating `.` for Erlang. Elixir's
+  `do ... end` blocks aren't recognized, so multi-line Elixir goes
+  through `--load`.
 - Ctrl-C cancels the line being edited but doesn't send the `interrupt`
   op during a running evaluation yet.
 

bin/dune

@@ -1,5 +1,10 @@
+(library
+ (name repl_input)
+ (modules repl_input))
+
 (executable
  (name main)
  (public_name mezcaml)
  (package mezcaml)
- (libraries mezcaml linenoise unix dune-build-info))
+ (modules main)
+ (libraries mezcaml repl_input linenoise unix dune-build-info))

bin/main.ml

@@ -92,22 +92,12 @@ let print_response msg =
   flush stdout;
   flush stderr
 
-let server_description conn =
-  let versions =
-    List.find_map
-      (fun msg -> Mezcaml.Bencode.get msg "versions")
-      (Mezcaml.Client.describe conn)
-  in
-  match versions with
-    | Some (Mezcaml.Bencode.Dict entries) ->
-        entries
-        |> List.filter_map
-             (fun (name, v) ->
-                match Mezcaml.Bencode.get_string v "version-string" with
-                  | Some version -> Some (name ^ " " ^ version)
-                  | None -> None)
-        |> String.concat ", "
-    | _ -> "unknown server"
+let server_description versions =
+  if versions = [] then "unknown server"
+  else
+    versions
+    |> List.map (fun (name, v) -> if v = "" then name else name ^ " " ^ v)
+    |> String.concat ", "
 
 (* +-----------------------------------------------------------------+
    | Interactive REPL                                                |
@@ -142,48 +132,72 @@ let setup_line_editor conn =
          (try mkdir_p (Filename.dirname file) with Unix.Unix_error _ -> ());
          ignore (LNoise.history_load ~filename:file : (unit, string) result)
      | None -> ());
-  (* Tab completion, powered by the server. The server completes the
-     whole line and returns full-line candidates, so they can be
-     handed to linenoise as they come. A server hiccup must not take
-     down the line editor. *)
+  (* Tab completion: ask the server about the trailing token (see
+     Repl_input). A server hiccup must not take down the line editor. *)
   LNoise.set_completion_callback
     (fun line completions ->
-       if String.trim line <> "" then
-         match Mezcaml.Client.completions conn line with
+       let head, token = Repl_input.split_for_completion line in
+       if token <> "" then
+         match Mezcaml.Client.completions conn token with
            | candidates ->
-               List.iter (LNoise.add_completion completions) candidates
+               List.iter
+                 (fun candidate ->
+                    LNoise.add_completion completions (head ^ candidate))
+                 candidates
            | exception _ -> ());
   history
 
 let repl conn =
+  let versions = Mezcaml.Client.versions (Mezcaml.Client.describe conn) in
   (* The banner is for interactive users; keep scripted (piped) runs
      clean. *)
   if Unix.isatty Unix.stdin then begin
     Printf.printf "mezcaml %s, a minimal nREPL client. Ctrl-D to quit.\n" (version ());
-    Printf.printf "Connected to %s:%d (%s)\n" !host !port (server_description conn)
+    Printf.printf "Connected to %s:%d (%s)\n" !host !port (server_description versions)
   end;
+  let dialect = Repl_input.dialect_of_versions (List.map fst versions) in
   let history = setup_line_editor conn in
   let remember line =
-    ignore (LNoise.history_add line : (unit, string) result);
+    ignore (LNoise.history_add line : (unit, string) result)
+  in
+  (* Saved once per form rather than per line: linenoise rewrites
+     the whole file on every save, which adds up when a big form is
+     pasted in. *)
+  let save_history () =
     match history with
       | Some file ->
           ignore (LNoise.history_save ~filename:file : (unit, string) result)
       | None -> ()
   in
-  let rec loop () =
-    match LNoise.linenoise "mezcaml> " with
+  (* Keep reading continuation lines until the input amounts to a
+     whole form. Each physical line is remembered separately because
+     the history file is line-based. *)
+  let rec read_form acc =
+    let prompt = if acc = [] then "mezcaml> " else "    ...> " in
+    match LNoise.linenoise prompt with
       | exception Sys.Break ->
-          (* Ctrl-C cancels the current line, not the session. *)
+          (* Ctrl-C cancels the input being edited, not the session. *)
           print_newline ();
-          loop ()
-      | None ->
+          read_form []
+      | None -> `Eof
+      | Some "" when acc = [] -> read_form []
+      | Some line ->
+          (* Blank continuation lines aren't worth a history entry. *)
+          if line <> "" then remember line;
+          let acc = line :: acc in
+          let form = String.concat "\n" (List.rev acc) in
+          if Repl_input.complete dialect form then `Form form
+          else read_form acc
+  in
+  let rec loop () =
+    match read_form [] with
+      | `Eof ->
+          save_history ();
           print_newline ();
           ignore (Mezcaml.Client.close_session conn : Mezcaml.Bencode.t list)
-      | Some "" ->
-          loop ()
-      | Some line ->
-          remember line;
-          ignore (Mezcaml.Client.eval conn ~f:print_response line
+      | `Form form ->
+          save_history ();
+          ignore (Mezcaml.Client.eval conn ~f:print_response form
                   : Mezcaml.Bencode.t list);
           loop ()
   in

bin/repl_input.ml

@@ -0,0 +1,108 @@
+(* Delimiters end the token being completed. Besides whitespace,
+   brackets and quotes, the set covers prefix syntax that sits
+   directly against a symbol: Clojure's @deref, ^meta, ~unquote,
+   #'var and `syntax-quote, Elixir's %Struct and &capture. Module
+   separators (lists:reverse, clojure.string, str/join, Enum.map)
+   stay in the token. One dialect-agnostic set on purpose: a
+   mis-split token only costs a completion the server declines, so
+   unlike form completeness below it earns no per-dialect
+   machinery. *)
+let is_delimiter = function
+  | ' ' | '\t' | '(' | ')' | '[' | ']' | '{' | '}'
+  | '"' | '\'' | '`' | ',' | ';' | '@' | '#' | '^' | '~' | '\\'
+  | '&' | '%' -> true
+  | _ -> false
+
+let split_for_completion line =
+  let n = String.length line in
+  let rec token_start i =
+    if i = 0 || is_delimiter line.[i - 1] then i else token_start (i - 1)
+  in
+  let s = token_start n in
+  (String.sub line 0 s, String.sub line s (n - s))
+
+type dialect = Lisp | Erlang | Generic
+
+(* Elixir is checked before Erlang: repartee reports both, and the
+   dot rule must not win there. Elixir's do/end blocks can't be
+   bracket-counted either, so it falls back to Generic. *)
+let dialect_of_versions names =
+  let has name = List.mem name names in
+  if has "clojure" || has "babashka" || has "basilisp" then Lisp
+  else if has "elixir" then Generic
+  else if has "erlang" then Erlang
+  else Generic
+
+(* Helpers shared by the scanners below. [scan_quoted] returns the
+   position just past the closing quote, or [None] when the input
+   ends inside the quoted region (or on a dangling escape), in which
+   case the form cannot be complete. *)
+let rec skip_line input i =
+  if i >= String.length input || input.[i] = '\n' then i
+  else skip_line input (i + 1)
+
+let rec scan_quoted input i quote =
+  if i >= String.length input then None
+  else
+    match input.[i] with
+      | '\\' ->
+          if i + 1 >= String.length input then None
+          else scan_quoted input (i + 2) quote
+      | c when c = quote -> Some (i + 1)
+      | _ -> scan_quoted input (i + 1) quote
+
+(* Bracket/string scanner shared by the Lisp and Generic rules. With
+   [lisp], ;-comments hide the rest of the line and a backslash
+   starts a character literal (\a, \(, \newline). *)
+let balanced ~lisp input =
+  let n = String.length input in
+  let rec scan i depth =
+    if i >= n then Some depth
+    else
+      match input.[i] with
+        | ';' when lisp -> scan (skip_line input (i + 1)) depth
+        | '\\' when lisp -> if i + 1 >= n then None else scan (i + 2) depth
+        | '"' ->
+            (match scan_quoted input (i + 1) '"' with
+               | Some j -> scan j depth
+               | None -> None)
+        | '(' | '[' | '{' -> scan (i + 1) (depth + 1)
+        | ')' | ']' | '}' -> scan (i + 1) (depth - 1)
+        | _ -> scan (i + 1) depth
+  in
+  match scan 0 0 with
+    | Some depth -> depth <= 0
+    | None -> false
+
+(* Erlang forms end with a dot. The dot flag set here survives only
+   across whitespace and %-comments to the end of the input, so 3.14
+   is not a terminator, and neither is a dot inside a string, a
+   quoted atom or a [$c] character literal. *)
+let erlang_complete input =
+  let n = String.length input in
+  let rec scan i depth dot =
+    if i >= n then Some (depth, dot)
+    else
+      match input.[i] with
+        | '%' -> scan (skip_line input (i + 1)) depth dot
+        | '$' -> if i + 1 >= n then None else scan (i + 2) depth false
+        | ('"' | '\'') as quote ->
+            (match scan_quoted input (i + 1) quote with
+               | Some j -> scan j depth false
+               | None -> None)
+        | '(' | '[' | '{' -> scan (i + 1) (depth + 1) false
+        | ')' | ']' | '}' -> scan (i + 1) (depth - 1) false
+        | '.' -> scan (i + 1) depth true
+        | ' ' | '\t' | '\n' -> scan (i + 1) depth dot
+        | _ -> scan (i + 1) depth false
+  in
+  match scan 0 0 false with
+    | Some (depth, dot) -> dot && depth <= 0
+    | None -> false
+
+let complete dialect input =
+  String.trim input <> ""
+  && (match dialect with
+        | Lisp -> balanced ~lisp:true input
+        | Generic -> balanced ~lisp:false input
+        | Erlang -> erlang_complete input)