Handle cursor completion inside fn/rescue/catch/else/after

Author: josevalim

lib/eex/lib/eex/compiler.ex

@@ -71,10 +71,10 @@ defmodule EEx.Compiler do
       {:ok, expr, new_line, new_column, rest} ->
         {key, expr} =
           case :elixir_tokenizer.tokenize(expr, 1, file: "eex", check_terminators: false) do
-            {:ok, _line, _column, _warnings, tokens} ->
+            {:ok, _line, _column, _warnings, rev_tokens, []} ->
               # We ignore warnings because the code will be tokenized
               # again later with the right line+column info
-              token_key(tokens, expr)
+              token_key(rev_tokens, expr)
 
             {:error, _, _, _, _} ->
               {:expr, expr}
@@ -164,8 +164,8 @@ defmodule EEx.Compiler do
   end
 
   # Receives tokens and check if it is a start, middle or an end token.
-  defp token_key(tokens, expr) do
-    case {tokens, tokens |> Enum.reverse() |> drop_eol()} do
+  defp token_key(rev_tokens, expr) do
+    case {Enum.reverse(rev_tokens), drop_eol(rev_tokens)} do
       {[{:end, _} | _], [{:do, _} | _]} ->
         {:middle_expr, expr}
 

lib/elixir/lib/code/fragment.ex

@@ -481,10 +481,10 @@ defmodule Code.Fragment do
 
   defp operator(rest, count, acc, _call_op?) do
     case :elixir_tokenizer.tokenize(acc, 1, 1, []) do
-      {:ok, _, _, _, [{:atom, _, _}]} ->
+      {:ok, _, _, _, [{:atom, _, _}], []} ->
         {{:unquoted_atom, tl(acc)}, count}
 
-      {:ok, _, _, _, [{_, _, op}]} ->
+      {:ok, _, _, _, [{_, _, op}], []} ->
         {rest, dot_count} = strip_spaces(rest, count)
 
         cond do
@@ -1088,9 +1088,41 @@ defmodule Code.Fragment do
   @spec container_cursor_to_quoted(List.Chars.t(), keyword()) ::
           {:ok, Macro.t()} | {:error, {location :: keyword, binary | {binary, binary}, binary}}
   def container_cursor_to_quoted(fragment, opts \\ []) do
-    opts =
-      Keyword.take(opts, [:file, :line, :column, :columns, :token_metadata, :literal_encoder])
+    opts = Keyword.take(opts, [:columns, :token_metadata, :literal_encoder])
+    opts = [cursor_completion: true, emit_warnings: false] ++ opts
+
+    file = Keyword.get(opts, :file, "nofile")
+    line = Keyword.get(opts, :line, 1)
+    column = Keyword.get(opts, :column, 1)
+
+    case :elixir_tokenizer.tokenize(to_charlist(fragment), line, column, opts) do
+      {:ok, line, column, _warnings, rev_tokens, rev_terminators} ->
+        tokens = :lists.reverse(rev_tokens, rev_terminators)
+
+        case :elixir.tokens_to_quoted(tokens, file, opts) do
+          {:ok, ast} ->
+            {:ok, ast}
+
+          {:error, error} ->
+            # In case parsing fails, we give it another shot but handling fn/do/else/catch/rescue/after.
+            tokens =
+              :lists.reverse(
+                rev_tokens,
+                [{:stab_op, {line, column, nil}, :->}, {nil, {line, column + 2, nil}}] ++
+                  Enum.map(rev_terminators, fn tuple ->
+                    {line, column, info} = elem(tuple, 1)
+                    put_elem(tuple, 1, {line, column + 5, info})
+                  end)
+              )
+
+            case :elixir.tokens_to_quoted(tokens, file, opts) do
+              {:ok, ast} -> {:ok, ast}
+              {:error, _} -> {:error, error}
+            end
+        end
 
-    Code.string_to_quoted(fragment, [cursor_completion: true, emit_warnings: false] ++ opts)
+      {:error, info, _rest, _warnings, _so_far} ->
+        {:error, :elixir.format_token_error(info)}
+    end
   end
 end

lib/elixir/src/elixir.erl

@@ -7,7 +7,8 @@
 -export([
   string_to_tokens/5, tokens_to_quoted/3, 'string_to_quoted!'/5,
   env_for_eval/1, quoted_to_erl/2, eval_forms/3, eval_quoted/3,
-  eval_quoted/4, eval_local_handler/2, eval_external_handler/3
+  eval_quoted/4, eval_local_handler/2, eval_external_handler/3,
+  format_token_error/1
 ]).
 -include("elixir.hrl").
 -define(system, 'Elixir.System').
@@ -459,18 +460,21 @@ quoted_to_erl(Quoted, ErlS, ExS, Env) ->
 
 string_to_tokens(String, StartLine, StartColumn, File, Opts) when is_integer(StartLine), is_binary(File) ->
   case elixir_tokenizer:tokenize(String, StartLine, StartColumn, Opts) of
-    {ok, _Line, _Column, [], Tokens} ->
-      {ok, Tokens};
-    {ok, _Line, _Column, Warnings, Tokens} ->
+    {ok, _Line, _Column, [], Tokens, Terminators} ->
+      {ok, lists:reverse(Tokens, Terminators)};
+    {ok, _Line, _Column, Warnings, Tokens, Terminators} ->
       (lists:keyfind(emit_warnings, 1, Opts) /= {emit_warnings, false}) andalso
         [elixir_errors:erl_warn(L, File, M) || {L, M} <- lists:reverse(Warnings)],
-      {ok, Tokens};
-    {error, {Location, {ErrorPrefix, ErrorSuffix}, Token}, _Rest, _Warnings, _SoFar} ->
-      {error, {Location, {to_binary(ErrorPrefix), to_binary(ErrorSuffix)}, to_binary(Token)}};
-    {error, {Location, Error, Token}, _Rest, _Warnings, _SoFar} ->
-      {error, {Location, to_binary(Error), to_binary(Token)}}
+      {ok, lists:reverse(Tokens, Terminators)};
+    {error, Info, _Rest, _Warnings, _SoFar} ->
+      {error, format_token_error(Info)}
   end.
 
+format_token_error({Location, {ErrorPrefix, ErrorSuffix}, Token}) ->
+  {Location, {to_binary(ErrorPrefix), to_binary(ErrorSuffix)}, to_binary(Token)};
+format_token_error({Location, Error, Token}) ->
+  {Location, to_binary(Error), to_binary(Token)}.
+
 tokens_to_quoted(Tokens, WarningFile, Opts) ->
   handle_parsing_opts(WarningFile, Opts),
 

lib/elixir/src/elixir_interpolation.erl

@@ -62,11 +62,11 @@ extract([$#, ${ | Rest], Buffer, Output, Line, Column, Scope, true, Last) ->
       extract(NewRest, [], Output2, EndLine, EndColumn + 1, NewScope, true, Last);
     {error, Reason, _, _, _} ->
       {error, Reason};
-    {ok, EndLine, EndColumn, Warnings, Tokens} when Scope#elixir_tokenizer.cursor_completion /= false ->
+    {ok, EndLine, EndColumn, Warnings, Tokens, Terminators} when Scope#elixir_tokenizer.cursor_completion /= false ->
       NewScope = Scope#elixir_tokenizer{warnings=Warnings, cursor_completion=noprune},
-      Output2 = build_interpol(Line, Column, EndLine, EndColumn, Tokens, Output1),
+      Output2 = build_interpol(Line, Column, EndLine, EndColumn, lists:reverse(Tokens, Terminators), Output1),
       extract([], [], Output2, EndLine, EndColumn, NewScope, true, Last);
-    {ok, _, _, _, _} ->
+    {ok, _, _, _, _, _} ->
       {error, {string, Line, Column, "missing interpolation terminator: \"}\"", []}}
   end;
 

lib/elixir/src/elixir_tokenizer.erl

@@ -138,12 +138,12 @@ tokenize(String, Line, Opts) ->
 tokenize([], Line, Column, #elixir_tokenizer{cursor_completion=Cursor} = Scope, Tokens) when Cursor /= false ->
   #elixir_tokenizer{ascii_identifiers_only=Ascii, terminators=Terminators, warnings=Warnings} = Scope,
 
-  {CursorColumn, CursorTerminators, CursorTokens} =
+  {CursorColumn, CursorTerminators, AccTokens} =
     add_cursor(Line, Column, Cursor, Terminators, Tokens),
 
   AllWarnings = maybe_unicode_lint_warnings(Ascii, Tokens, Warnings),
-  AccTokens = cursor_complete(Line, CursorColumn, CursorTerminators, CursorTokens),
-  {ok, Line, Column, AllWarnings, AccTokens};
+  {AccTerminators, _AccColumn} = cursor_complete(Line, CursorColumn, CursorTerminators),
+  {ok, Line, CursorColumn, AllWarnings, AccTokens, AccTerminators};
 
 tokenize([], EndLine, EndColumn, #elixir_tokenizer{terminators=[{Start, {StartLine, StartColumn, _}, _} | _]} = Scope, Tokens) ->
   End = terminator(Start),
@@ -163,7 +163,7 @@ tokenize([], EndLine, EndColumn, #elixir_tokenizer{terminators=[{Start, {StartLi
 tokenize([], Line, Column, #elixir_tokenizer{} = Scope, Tokens) ->
   #elixir_tokenizer{ascii_identifiers_only=Ascii, warnings=Warnings} = Scope,
   AllWarnings = maybe_unicode_lint_warnings(Ascii, Tokens, Warnings),
-  {ok, Line, Column, AllWarnings, lists:reverse(Tokens)};
+  {ok, Line, Column, AllWarnings, Tokens, []};
 
 % VC merge conflict
 
@@ -1728,19 +1728,15 @@ error(Reason, Rest, #elixir_tokenizer{warnings=Warnings}, Tokens) ->
 
 %% Cursor handling
 
-cursor_complete(Line, Column, Terminators, Tokens) ->
-  {AccTokens, _} =
-    lists:foldl(
-      fun({Start, _, _}, {NewTokens, NewColumn}) ->
-        End = terminator(Start),
-        AccTokens = [{End, {Line, NewColumn, nil}} | NewTokens],
-        AccColumn = NewColumn + length(erlang:atom_to_list(End)),
-        {AccTokens, AccColumn}
-      end,
-      {Tokens, Column},
-      Terminators
-    ),
-  lists:reverse(AccTokens).
+cursor_complete(Line, Column, Terminators) ->
+  lists:mapfoldl(
+    fun({Start, _, _}, AccColumn) ->
+      End = terminator(Start),
+      {{End, {Line, AccColumn, nil}}, AccColumn + length(erlang:atom_to_list(End))}
+    end,
+    Column,
+    Terminators
+  ).
 
 add_cursor(_Line, Column, noprune, Terminators, Tokens) ->
   {Column, Terminators, Tokens};
@@ -1783,8 +1779,6 @@ prune_tokens([{'{', _} | Tokens], ['}' | Opener], Terminators) ->
 prune_tokens([{'<<', _} | Tokens], ['>>' | Opener], Terminators) ->
   prune_tokens(Tokens, Opener, Terminators);
 %%% Handle anonymous functions
-prune_tokens(Tokens, [], [{'fn', _, _} | Terminators]) ->
-  prune_tokens(drop_including(Tokens, 'fn'), [], Terminators);
 prune_tokens([{'(', _}, {capture_op, _, _} | Tokens], [], [{'(', _, _} | Terminators]) ->
   prune_tokens(Tokens, [], Terminators);
 %%% or it is time to stop...
@@ -1794,6 +1788,8 @@ prune_tokens([{'eol', _} | _] = Tokens, [], Terminators) ->
   {Tokens, Terminators};
 prune_tokens([{',', _} | _] = Tokens, [], Terminators) ->
   {Tokens, Terminators};
+prune_tokens([{'fn', _} | _] = Tokens, [], Terminators) ->
+  {Tokens, Terminators};
 prune_tokens([{'do', _} | _] = Tokens, [], Terminators) ->
   {Tokens, Terminators};
 prune_tokens([{'(', _} | _] = Tokens, [], Terminators) ->
@@ -1827,7 +1823,3 @@ prune_tokens([_ | Tokens], Opener, Terminators) ->
   prune_tokens(Tokens, Opener, Terminators);
 prune_tokens([], [], Terminators) ->
   {[], Terminators}.
-
-drop_including([{Token, _} | Tokens], Token) -> Tokens;
-drop_including([_ | Tokens], Token) -> drop_including(Tokens, Token);
-drop_including([], _Token) -> [].

lib/elixir/test/elixir/code_fragment_test.exs

@@ -1158,9 +1158,7 @@ defmodule CodeFragmentTest do
       assert cc2q!("foo(bar do baz ") == s2q!("foo(bar do baz(__cursor__()) end)")
       assert cc2q!("foo(bar do baz(") == s2q!("foo(bar do baz(__cursor__()) end)")
       assert cc2q!("foo(bar do baz bat,") == s2q!("foo(bar do baz(bat, __cursor__()) end)")
-
-      assert {:error, {_, "syntax error before: ", "'end'"}} =
-               CF.container_cursor_to_quoted("foo(bar do baz, bat")
+      assert cc2q!("foo(bar do baz, bat") == s2q!("foo(bar do baz, __cursor__() -> nil end)")
     end
 
     test "keyword lists" do
@@ -1232,11 +1230,11 @@ defmodule CodeFragmentTest do
     end
 
     test "removes anonymous functions" do
-      assert cc2q!("(fn") == s2q!("(__cursor__())")
-      assert cc2q!("(fn x") == s2q!("(__cursor__())")
-      assert cc2q!("(fn x ->") == s2q!("(__cursor__())")
-      assert cc2q!("(fn x -> x") == s2q!("(__cursor__())")
-      assert cc2q!("(fn x, y -> x + y") == s2q!("(__cursor__())")
+      assert cc2q!("(fn") == s2q!("(fn __cursor__() -> nil end)")
+      assert cc2q!("(fn x") == s2q!("(fn __cursor__() -> nil end)")
+      assert cc2q!("(fn x ->") == s2q!("(fn x -> __cursor__() end)")
+      assert cc2q!("(fn x -> x") == s2q!("(fn x -> __cursor__() end)")
+      assert cc2q!("(fn x, y -> x + y") == s2q!("(fn x, y -> x + __cursor__() end)")
       assert cc2q!("(fn x, y -> x + y end") == s2q!("(__cursor__())")
     end
 

lib/elixir/test/erlang/tokenizer_test.erl

@@ -5,8 +5,8 @@ tokenize(String) ->
   tokenize(String, []).
 
 tokenize(String, Opts) ->
-  {ok, _Line, _Column, _Warnings, Result} = elixir_tokenizer:tokenize(String, 1, Opts),
-  Result.
+  {ok, _Line, _Column, _Warnings, Result, []} = elixir_tokenizer:tokenize(String, 1, Opts),
+  lists:reverse(Result).
 
 tokenize_error(String) ->
   {error, Error, _, _, _} = elixir_tokenizer:tokenize(String, 1, []),