From 9b9e30843dc5244a8cde57529543af01b8727a64 Mon Sep 17 00:00:00 2001 From: doorgan Date: Mon, 17 Mar 2025 13:38:07 -0300 Subject: [PATCH 1/7] Merge comments into the AST --- lib/elixir/lib/code.ex | 47 +- lib/elixir/lib/code/comments.ex | 589 ++++++++++++++++ .../test/elixir/code/ast_comments_test.exs | 651 ++++++++++++++++++ .../code_formatter/ast_comments_test.exs | 521 ++++++++++++++ 4 files changed, 1802 insertions(+), 6 deletions(-) create mode 100644 lib/elixir/lib/code/comments.ex create mode 100644 lib/elixir/test/elixir/code/ast_comments_test.exs create mode 100644 lib/elixir/test/elixir/code_formatter/ast_comments_test.exs diff --git a/lib/elixir/lib/code.ex b/lib/elixir/lib/code.ex index 9513b30bb8..8665e3da75 100644 --- a/lib/elixir/lib/code.ex +++ b/lib/elixir/lib/code.ex @@ -1036,12 +1036,13 @@ defmodule Code do [ unescape: false, literal_encoder: &{:ok, {:__block__, &2, [&1]}}, + include_comments: true, token_metadata: true, emit_warnings: false ] ++ opts - {forms, comments} = string_to_quoted_with_comments!(string, to_quoted_opts) - to_algebra_opts = [comments: comments] ++ opts + forms = string_to_quoted!(string, to_quoted_opts) + to_algebra_opts = opts doc = Code.Formatter.to_algebra(forms, to_algebra_opts) Inspect.Algebra.format(doc, line_length) end @@ -1254,11 +1255,22 @@ defmodule Code do file = Keyword.get(opts, :file, "nofile") line = Keyword.get(opts, :line, 1) column = Keyword.get(opts, :column, 1) + include_comments = Keyword.get(opts, :include_comments, false) - case :elixir.string_to_tokens(to_charlist(string), line, column, file, opts) do - {:ok, tokens} -> - :elixir.tokens_to_quoted(tokens, file, opts) + Process.put(:code_formatter_comments, []) + opts = [preserve_comments: &preserve_comments/5] ++ opts + with {:ok, tokens} <- :elixir.string_to_tokens(to_charlist(string), line, column, file, opts), + {:ok, quoted} <- :elixir.tokens_to_quoted(tokens, file, opts) do + if include_comments do + quoted = Code.Normalizer.normalize(quoted) + quoted = Code.Comments.merge_comments(quoted, Process.get(:code_formatter_comments)) + + {:ok, quoted} + else + {:ok, quoted} + end + else {:error, _error_msg} = error -> error end @@ -1280,7 +1292,30 @@ defmodule Code do file = Keyword.get(opts, :file, "nofile") line = Keyword.get(opts, :line, 1) column = Keyword.get(opts, :column, 1) - :elixir.string_to_quoted!(to_charlist(string), line, column, file, opts) + include_comments = Keyword.get(opts, :include_comments, false) + + Process.put(:code_formatter_comments, []) + + opts = + if include_comments do + [preserve_comments: &preserve_comments/5, + literal_encoder: &{:ok, {:__block__, &2, [&1]}}, + token_metadata: true, + unescape: false, + columns: true, +] ++ opts + else + opts + end + + quoted = :elixir.string_to_quoted!(to_charlist(string), line, column, file, opts) + + if include_comments do + # quoted = Code.Normalizer.normalize(quoted) + Code.Comments.merge_comments(quoted, Process.get(:code_formatter_comments)) + else + quoted + end end @doc """ diff --git a/lib/elixir/lib/code/comments.ex b/lib/elixir/lib/code/comments.ex new file mode 100644 index 0000000000..d97e05b21b --- /dev/null +++ b/lib/elixir/lib/code/comments.ex @@ -0,0 +1,589 @@ +defmodule Code.Comments do + @moduledoc false + + @end_fields [:end, :closing, :end_of_expression] + @block_names [:do, :else, :catch, :rescue, :after] + @arrow_ops [:|>, :<<<, :>>>, :<~, :~>, :<<~, :~>>, :<~>, :"<|>", :->] + + defguardp is_arrow_op(op) when is_atom(op) and op in @arrow_ops + + @doc """ + Merges the comments into the given quoted expression. + + There are three types of comments: + - `:leading_comments`: Comments that are located before a node, + or in the same line. + + Examples: + + # This is a leading comment + foo # This one too + + - `:trailing_comments`: Comments that are located after a node, and + before the end of the parent enclosing the node(or the root document). + + Examples: + + foo + # This is a trailing comment + # This one too + + - `:inner_comments`: Comments that are located inside an empty node. + + Examples: + + foo do + # This is an inner comment + end + + [ + # This is an inner comment + ] + + %{ + # This is an inner comment + } + + A comment may be considered inner or trailing depending on wether the enclosing + node is empty or not. For example, in the following code: + + foo do + # This is an inner comment + end + + The comment is considered inner because the `do` block is empty. However, in the + following code: + + foo do + bar + # This is a trailing comment + end + + The comment is considered trailing to `bar` because the `do` block is not empty. + + In the case no nodes are present in the AST but there are comments, they are + inserted into a placeholder `:__block__` node as `:inner_comments`. + """ + @spec merge_comments(Macro.t(), list(map)) :: Macro.t() + def merge_comments({:__block__, _, []} = empty_ast, comments) do + comments = Enum.sort_by(comments, & &1.line) + put_comments(empty_ast, :inner_comments, comments) + end + def merge_comments(quoted, comments) do + comments = Enum.sort_by(comments, & &1.line) + + state = %{ + comments: comments, + parent_doend_meta: [] + } + + {quoted, %{comments: leftovers}} = Macro.prewalk(quoted, state, &do_merge_comments/2) + + merge_leftovers(quoted, leftovers) + end + + defp merge_leftovers({:__block__, _, args} = quoted, comments) when is_list(args) do + {last_arg, args} = List.pop_at(args, -1) + + case last_arg do + nil -> + append_comments(quoted, :inner_comments, comments) + {_, _, _} = last_arg -> + last_arg = append_comments(last_arg, :trailing_comments, comments) + + args = args ++ [last_arg] + put_args(quoted, args) + + _ -> + append_comments(quoted, :trailing_comments, comments) + end + end + + defp merge_leftovers(quoted, comments) do + append_comments(quoted, :trailing_comments, comments) + end + + defp do_merge_comments({_, _, _} = quoted, state) do + {quoted, state} = merge_trailing_comments(quoted, state) + merge_leading_comments(quoted, state) + end + + defp do_merge_comments(quoted, state) do + {quoted, state} + end + + defp merge_leading_comments(quoted, state) do + # If a comment is placed on top of a pipeline or binary operator line, + # we should not merge it with the operator itself. Instead, we should + # merge it with the first argument of the pipeline. + # + # This avoids the comment being moved up when formatting the code. + with {form, _, _} <- quoted, + false <- is_arrow_op(form), + :error <- Code.Identifier.binary_op(form) do + {comments, rest} = gather_leading_comments_for_node(quoted, state.comments) + comments = Enum.sort_by(comments, & &1.line) + + quoted = put_comments(quoted, :leading_comments, comments) + {quoted, %{state | comments: rest}} + else + _ -> + {quoted, state} + end + end + + defp gather_leading_comments_for_node(quoted, comments) do + line = get_line(quoted, 0) + + {comments, rest} = + Enum.reduce(comments, {[], []}, fn + comment, {comments, rest} -> + if comment.line <= line do + {[comment | comments], rest} + else + {comments, [comment | rest]} + end + end) + + {comments, rest} + end + + # Structs + defp merge_trailing_comments({:%, _, [name, args]} = quoted, state) do + {args, comments} = merge_trailing_comments(args, state.comments) + + quoted = put_args(quoted, [name, args]) + + {quoted, %{state | comments: comments}} + end + + # Maps + defp merge_trailing_comments({:%{}, _, [{_key, _value} | _] = args} = quoted, %{comments: comments} = state) do + case List.pop_at(args, -1) do + {{last_key, last_value}, args} -> + start_line = get_line(last_value) + end_line = get_end_line(quoted, start_line) + + {trailing_comments, comments} = + Enum.split_with(comments, & &1.line > start_line and &1.line < end_line) + + last_value = append_comments(last_value, :trailing_comments, trailing_comments) + + args = args ++ [{last_key, last_value}] + + quoted = put_args(quoted, args) + {quoted, %{state | comments: comments}} + + {{:unquote_splicing, _, _} = unquote_splicing, other} -> + start_line = get_line(unquote_splicing) + end_line = get_end_line(quoted, start_line) + + {trailing_comments, comments} = + Enum.split_with(comments, & &1.line > start_line and &1.line < end_line) + + unquote_splicing = append_comments(unquote_splicing, :trailing_comments, trailing_comments) + + args = other ++ [unquote_splicing] + quoted = put_args(quoted, args) + + {quoted, %{state | comments: comments}} + end + end + + # Lists + defp merge_trailing_comments({:__block__, _, [args]} = quoted, %{comments: comments} = state) when is_list(args) do + {quoted, comments} = + case List.pop_at(args, -1) do + {nil, _} -> + start_line = get_line(quoted) + end_line = get_end_line(quoted, start_line) + + {trailing_comments, comments} = + Enum.split_with(comments, & &1.line > start_line and &1.line < end_line) + + quoted = append_comments(quoted, :inner_comments, trailing_comments) + + {quoted, comments} + + {{last_key, last_value}, args} -> + start_line = get_line(last_value) + end_line = get_end_line(quoted, start_line) + + {trailing_comments, comments} = + Enum.split_with(comments, & &1.line > start_line and &1.line < end_line) + + last_value = append_comments(last_value, :trailing_comments, trailing_comments) + + args = args ++ [{last_key, last_value}] + + quoted = put_args(quoted, [args]) + {quoted, comments} + {{:unquote_splicing, _, _} = unquote_splicing, other} -> + start_line = get_line(unquote_splicing) + end_line = get_end_line(quoted, start_line) + + {trailing_comments, comments} = + Enum.split_with(comments, & &1.line > start_line and &1.line < end_line) + + unquote_splicing = append_comments(unquote_splicing, :trailing_comments, trailing_comments) + + args = other ++ [unquote_splicing] + quoted = put_args(quoted, [args]) + + {quoted, comments} + + {{_, _, _} = value, args} -> + start_line = get_line(value) + end_line = get_end_line(quoted, start_line) + + {trailing_comments, comments} = + Enum.split_with(comments, & &1.line > start_line and &1.line < end_line) + + value = append_comments(value, :trailing_comments, trailing_comments) + + args = args ++ [value] + quoted = put_args(quoted, [args]) + + {quoted, comments} + + _ -> + {quoted, comments} + end + + {quoted, %{state | parent_doend_meta: [], comments: comments}} + end + + + # 2-tuples + defp merge_trailing_comments({:__block__, _, [{left, right}]} = quoted, %{comments: comments} = state) when is_tuple(left) and is_tuple(right) do + start_line = get_line(right) + end_line = get_end_line(quoted, start_line) + + {trailing_comments, comments} = + Enum.split_with(comments, & &1.line > start_line and &1.line < end_line) + + right = append_comments(right, :trailing_comments, trailing_comments) + + quoted = put_args(quoted, [{left, right}]) + {quoted, %{state | comments: comments}} + end + + # Stabs + defp merge_trailing_comments({:->, _, [left, right]} = quoted, state) do + start_line = get_line(right) + end_line = get_end_line({:__block__, state.parent_doend_meta, [quoted]}, start_line) + + {right, comments} = + case right do + {:__block__, _, _} -> + merge_block_trailing_comments(right, start_line, end_line, state.comments) + + call -> + line = get_line(call) + {trailing_comments, comments} = + Enum.split_with(state.comments, & &1.line > line and &1.line < end_line) + + call = append_comments(call, :trailing_comments, trailing_comments) + + {call, comments} + end + + quoted = put_args(quoted, [left, right]) + + {quoted, %{state | comments: comments}} + end + + # Calls + defp merge_trailing_comments({_, meta, args} = quoted, %{comments: comments} = state) when is_list(args) and meta != [] do + start_line = get_line(quoted) + end_line = get_end_line(quoted, start_line) + {last_arg, args} = List.pop_at(args, -1) + + meta_keys = Keyword.keys(meta) + + state = + if Enum.any?([:do, :closing], &(&1 in meta_keys)) do + %{state | parent_doend_meta: meta} + else + state + end + + {quoted, comments} = + case last_arg do + [{{:__block__, _, [name]}, _block_args} | _] = blocks when name in @block_names -> + {reversed_blocks, comments} = each_merge_named_block_trailing_comments(blocks, quoted, comments, []) + + last_arg = Enum.reverse(reversed_blocks) + + args = args ++ [last_arg] + quoted = put_args(quoted, args) + + {quoted, comments} + + [{_key, _value} | _] = pairs -> + # Partial keyword list + {{last_key, last_value}, pairs} = List.pop_at(pairs, -1) + line = get_line(last_value) + + {trailing_comments, comments} = + Enum.split_with(comments, & &1.line > line and &1.line < end_line) + + last_value = append_comments(last_value, :trailing_comments, trailing_comments) + + pairs = pairs ++ [{last_key, last_value}] + + args = args ++ [pairs] + + quoted = put_args(quoted, args) + + {quoted, comments} + + {form, _, _} when form != :-> -> + line = get_line(last_arg) + {trailing_comments, comments} = + Enum.split_with(comments, & &1.line > line and &1.line < end_line) + + last_arg = append_comments(last_arg, :trailing_comments, trailing_comments) + + args = args ++ [last_arg] + quoted = put_args(quoted, args) + + {quoted, comments} + nil -> + {trailing_comments, comments} = + Enum.split_with(comments, & &1.line > start_line and &1.line < end_line) + + quoted = append_comments(quoted, :inner_comments, trailing_comments) + {quoted, comments} + + _ -> + {quoted, comments} + end + + {quoted, %{state | comments: comments}} + end + + defp merge_trailing_comments(quoted, state) do + {quoted, state} + end + + defp each_merge_named_block_trailing_comments([], _, comments, acc), do: {acc, comments} + + defp each_merge_named_block_trailing_comments([{block, block_args} | rest], parent, comments, acc) do + block_start = get_line(block) + block_end = + case rest do + [{next_block, _} | _] -> + get_line(next_block) + [] -> + get_end_line(parent, 0) + end + + {block, block_args, comments} = merge_named_block_trailing_comments(block, block_args, block_start, block_end, comments) + + acc = [{block, block_args} | acc] + + each_merge_named_block_trailing_comments(rest, parent, comments, acc) + end + + defp merge_named_block_trailing_comments(block, {_, _, args} = block_args, block_start, block_end, comments) when is_list(args) do + {last_arg, args} = List.pop_at(args, -1) + + case last_arg do + nil -> + {trailing_comments, comments} = + Enum.split_with(comments, & &1.line > block_start and &1.line < block_end) + + block_args = append_comments(block_args, :inner_comments, trailing_comments) + + {block, block_args, comments} + + last_arg when not is_list(last_arg) -> + {last_arg, comments} = + merge_trailing_comments_to_last_arg(last_arg, block_start, block_end, comments) + + args = args ++ [last_arg] + block_args = put_args(block_args, args) + + {block, block_args, comments} + + _ -> + {block, block_args, comments} + end + end + + # If a do/end block has a single argument, it will not be wrapped in a `:__block__` node, + # so we need to check for that. + defp merge_named_block_trailing_comments(block, {_, _, ctx} = single_arg, block_start, block_end, comments) when not is_list(ctx) do + {last_arg, comments} = + merge_trailing_comments_to_last_arg(single_arg, block_start, block_end, comments) + + {block, last_arg, comments} + end + + defp merge_named_block_trailing_comments(block, block_args, _, _, comments), + do: {block, block_args, comments} + + defp merge_block_trailing_comments({:__block__, _, args} = block, block_start, block_end, comments) do + {last_arg, args} = List.pop_at(args, -1) + + case last_arg do + nil -> + {trailing_comments, comments} = + Enum.split_with(comments, & &1.line > block_start and &1.line < block_end) + + trailing_comments = Enum.sort_by(trailing_comments, & &1.line) + + block = append_comments(block, :inner_comments, trailing_comments) + + {block, comments} + + last_arg when not is_list(last_arg) -> + {last_arg, comments} = + merge_trailing_comments_to_last_arg(last_arg, block_start, block_end, comments) + + args = args ++ [last_arg] + block = put_args(block, args) + + {block, comments} + + inner_list when is_list(inner_list) -> + {inner_list, comments} = + merge_trailing_comments_to_last_arg(inner_list, block_start, block_end, comments) + + args = args ++ [inner_list] + block = put_args(block, args) + + {block, comments} + + _ -> + {block, comments} + end + end + + defp merge_trailing_comments_to_last_arg(last_arg, block_start, block_end, comments) do + line = + case last_arg do + [] -> block_start + [first | _] -> get_line(first) + {_, _, _} -> get_line(last_arg) + _ -> block_start + end + + {trailing_comments, comments} = + Enum.split_with(comments, & &1.line > line and &1.line < block_end) + + last_arg = append_comments(last_arg, :trailing_comments, trailing_comments) + + {last_arg, comments} + end + + # ======= + + defp put_comments(quoted, key, comments) do + Macro.update_meta(quoted, &Keyword.put(&1, key, comments)) + end + + defp append_comments(quoted, key, comments) do + Macro.update_meta(quoted, fn meta -> + Keyword.update(meta, key, comments, &(&1 ++ comments)) + end) + end + + defp get_meta({_, meta, _}) when is_list(meta), do: meta + + + defp get_line({_, meta, _}, default \\ 1) + when is_list(meta) and (is_integer(default) or is_nil(default)) do + Keyword.get(meta, :line, default) + end + + defp get_end_line(quoted, default) when is_integer(default) do + get_end_position(quoted, line: default, column: 1)[:line] + end + + defp get_end_position(quoted, default) do + {_, position} = + Macro.postwalk(quoted, default, fn + {_, _, _} = quoted, end_position -> + current_end_position = get_node_end_position(quoted, default) + + end_position = + if compare_positions(end_position, current_end_position) == :gt do + end_position + else + current_end_position + end + + {quoted, end_position} + + terminal, end_position -> + {terminal, end_position} + end) + + position + end + + defp get_node_end_position(quoted, default) do + meta = get_meta(quoted) + + start_position = [ + line: meta[:line] || default[:line], + column: meta[:column] || default[:column] + ] + + get_meta(quoted) + |> Keyword.take(@end_fields) + |> Keyword.values() + |> Enum.map(fn end_field -> + position = Keyword.take(end_field, [:line, :column]) + + # If the node contains newlines, a newline is included in the + # column count. We subtract it so that the column represents the + # last non-whitespace character. + if Keyword.has_key?(end_field, :newlines) do + Keyword.update(position, :column, nil, &(&1 - 1)) + else + position + end + end) + |> Enum.concat([start_position]) + |> Enum.max_by( + & &1, + fn prev, next -> + compare_positions(prev, next) == :gt + end, + fn -> default end + ) + end + + defp compare_positions(left, right) do + left = coalesce_position(left) + right = coalesce_position(right) + + cond do + left == right -> + :eq + + left[:line] > right[:line] -> + :gt + + left[:line] == right[:line] and left[:column] > right[:column] -> + :gt + + true -> + :lt + end + end + + defp coalesce_position(position) do + line = position[:line] || 0 + column = position[:column] || 0 + + [line: line, column: column] + end + + defp put_args({form, meta, _args}, args) do + {form, meta, args} + end +end diff --git a/lib/elixir/test/elixir/code/ast_comments_test.exs b/lib/elixir/test/elixir/code/ast_comments_test.exs new file mode 100644 index 0000000000..bd4a2d2f74 --- /dev/null +++ b/lib/elixir/test/elixir/code/ast_comments_test.exs @@ -0,0 +1,651 @@ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Code.AstCommentsTest do + use ExUnit.Case, async: true + + def parse_string!(string) do + Code.string_to_quoted!(string, include_comments: true, emit_warnings: false) + end + + describe "merge_comments/2" do + test "merges comments in empty AST" do + quoted = + parse_string!(""" + # some comment + # another comment + """) + + assert {:__block__, meta, []} = quoted + + assert [%{line: 1, text: "# some comment"}, %{line: 2, text: "# another comment"}] = + meta[:inner_comments] + end + + test "merges leading comments in assorted terms" do + quoted = + parse_string!(""" + # leading var + var + # trailing var + """) + + assert {:var, meta, _} = quoted + + assert [%{line: 1, text: "# leading var"}] = meta[:leading_comments] + assert [%{line: 3, text: "# trailing var"}] = meta[:trailing_comments] + + quoted = + parse_string!(""" + # leading 1 + 1 + # trailing 1 + """) + + assert {:__block__, one_meta, [1]} = quoted + + assert [%{line: 1, text: "# leading 1"}] = one_meta[:leading_comments] + assert [%{line: 3, text: "# trailing 1"}] = one_meta[:trailing_comments] + + quoted = + parse_string!(""" + # leading qualified call + Foo.bar(baz) + # trailing qualified call + """) + + assert {{:., _, [_Foo, _bar]}, meta, _} = quoted + + assert [%{line: 1, text: "# leading qualified call"}] = meta[:leading_comments] + assert [%{line: 3, text: "# trailing qualified call"}] = meta[:trailing_comments] + + quoted = + parse_string!(""" + # leading qualified call + Foo. + # leading bar + bar(baz) + # trailing qualified call + """) + + assert {{:., _, [_Foo, _]}, meta, + [ + {:baz, _, _} + ]} = quoted + + assert [%{line: 1, text: "# leading qualified call"}, %{line: 3, text: "# leading bar"}] = + meta[:leading_comments] + + assert [%{line: 5, text: "# trailing qualified call"}] = meta[:trailing_comments] + end + + # Do/end blocks + + test "merges comments in do/end block" do + quoted = + parse_string!(""" + def a do + foo() + :ok + # A + end # B + """) + + assert {:def, def_meta, + [ + {:a, _, _}, + [ + {{:__block__, _, [:do]}, + {:__block__, _, + [ + {:foo, _, _}, + {:__block__, meta, [:ok]} + ]}} + ] + ]} = + quoted + + assert [%{line: 4, text: "# A"}] = meta[:trailing_comments] + + assert [%{line: 5, text: "# B"}] = def_meta[:trailing_comments] + end + + test "merges comments for named do/end blocks" do + quoted = + parse_string!(""" + def a do + # leading var1 + var1 + # trailing var1 + else + # leading var2 + var2 + # trailing var2 + catch + # leading var3 + var3 + # trailing var3 + rescue + # leading var4 + var4 + # trailing var4 + after + # leading var5 + var5 + # trailing var5 + end + """) + + assert {:def, _, + [ + {:a, _, _}, + [ + {{:__block__, _, [:do]}, {:var1, var1_meta, _}}, + {{:__block__, _, [:else]}, {:var2, var2_meta, _}}, + {{:__block__, _, [:catch]}, {:var3, var3_meta, _}}, + {{:__block__, _, [:rescue]}, {:var4, var4_meta, _}}, + {{:__block__, _, [:after]}, {:var5, var5_meta, _}} + ] + ]} = + quoted + + assert [%{line: 2, text: "# leading var1"}] = var1_meta[:leading_comments] + assert [%{line: 4, text: "# trailing var1"}] = var1_meta[:trailing_comments] + assert [%{line: 6, text: "# leading var2"}] = var2_meta[:leading_comments] + assert [%{line: 8, text: "# trailing var2"}] = var2_meta[:trailing_comments] + assert [%{line: 10, text: "# leading var3"}] = var3_meta[:leading_comments] + assert [%{line: 12, text: "# trailing var3"}] = var3_meta[:trailing_comments] + assert [%{line: 14, text: "# leading var4"}] = var4_meta[:leading_comments] + assert [%{line: 16, text: "# trailing var4"}] = var4_meta[:trailing_comments] + assert [%{line: 18, text: "# leading var5"}] = var5_meta[:leading_comments] + assert [%{line: 20, text: "# trailing var5"}] = var5_meta[:trailing_comments] + end + + test "merges inner comments for empty named do/end blocks" do + quoted = + parse_string!(""" + def a do + # inside do + else + # inside else + catch + # inside catch + rescue + # inside rescue + after + # inside after + end + """) + + assert {:def, _, + [ + {:a, _, _}, + [ + {{:__block__, _, [:do]}, {:__block__, do_meta, _}}, + {{:__block__, _, [:else]}, {:__block__, else_meta, _}}, + {{:__block__, _, [:catch]}, {:__block__, catch_meta, _}}, + {{:__block__, _, [:rescue]}, {:__block__, rescue_meta, _}}, + {{:__block__, _, [:after]}, {:__block__, after_meta, _}} + ] + ]} = + quoted + + assert [%{line: 2, text: "# inside do"}] = do_meta[:inner_comments] + assert [%{line: 4, text: "# inside else"}] = else_meta[:inner_comments] + assert [%{line: 6, text: "# inside catch"}] = catch_meta[:inner_comments] + assert [%{line: 8, text: "# inside rescue"}] = rescue_meta[:inner_comments] + assert [%{line: 10, text: "# inside after"}] = after_meta[:inner_comments] + end + + # Lists + + test "merges comments in list" do + quoted = + parse_string!(""" + [ + #leading 1 + 1, + #leading 2 + 2, + 3 + #trailing 3 + ] # trailing outside + """) + + assert {:__block__, list_meta, + [ + [ + {:__block__, one_meta, [1]}, + {:__block__, two_meta, [2]}, + {:__block__, three_meta, [3]} + ] + ]} = quoted + + assert [%{line: 2, text: "#leading 1"}] = one_meta[:leading_comments] + assert [%{line: 4, text: "#leading 2"}] = two_meta[:leading_comments] + assert [%{line: 7, text: "#trailing 3"}] = three_meta[:trailing_comments] + assert [%{line: 8, text: "# trailing outside"}] = list_meta[:trailing_comments] + end + + test "merges inner comments in empty list" do + quoted = + parse_string!(""" + [ + # inner 1 + # inner 2 + ] # trailing outside + """) + + assert {:__block__, list_meta, [[]]} = quoted + + assert [%{line: 2, text: "# inner 1"}, %{line: 3, text: "# inner 2"}] = + list_meta[:inner_comments] + + assert [%{line: 4, text: "# trailing outside"}] = list_meta[:trailing_comments] + end + + # Keyword lists + + test "merges comments in keyword list" do + quoted = + parse_string!(""" + [ + #leading a + a: 1, + #leading b + b: 2, + c: 3 + #trailing 3 + ] # trailing outside + """) + + assert {:__block__, keyword_list_meta, + [ + [ + { + {:__block__, a_key_meta, [:a]}, + {:__block__, _, [1]} + }, + { + {:__block__, b_key_meta, [:b]}, + {:__block__, _, [2]} + }, + { + {:__block__, _, [:c]}, + {:__block__, c_value_meta, [3]} + } + ] + ]} = quoted + + assert [%{line: 2, text: "#leading a"}] = a_key_meta[:leading_comments] + assert [%{line: 4, text: "#leading b"}] = b_key_meta[:leading_comments] + assert [%{line: 7, text: "#trailing 3"}] = c_value_meta[:trailing_comments] + assert [%{line: 8, text: "# trailing outside"}] = keyword_list_meta[:trailing_comments] + end + + test "merges comments in partial keyword list" do + quoted = + parse_string!(""" + [ + #leading 1 + 1, + #leading b + b: 2 + #trailing b + ] # trailing outside + """) + + assert {:__block__, keyword_list_meta, + [ + [ + {:__block__, one_key_meta, [1]}, + { + {:__block__, b_key_meta, [:b]}, + {:__block__, b_value_meta, [2]} + } + ] + ]} = quoted + + assert [%{line: 2, text: "#leading 1"}] = one_key_meta[:leading_comments] + assert [%{line: 4, text: "#leading b"}] = b_key_meta[:leading_comments] + assert [%{line: 6, text: "#trailing b"}] = b_value_meta[:trailing_comments] + assert [%{line: 7, text: "# trailing outside"}] = keyword_list_meta[:trailing_comments] + end + + # Tuples + + test "merges comments in n-tuple" do + quoted = + parse_string!(""" + { + #leading 1 + 1, + #leading 2 + 2, + 3 + #trailing 3 + } # trailing outside + """) + + assert {:{}, tuple_meta, + [ + {:__block__, one_meta, [1]}, + {:__block__, two_meta, [2]}, + {:__block__, three_meta, [3]} + ]} = quoted + + assert [%{line: 2, text: "#leading 1"}] = one_meta[:leading_comments] + assert [%{line: 4, text: "#leading 2"}] = two_meta[:leading_comments] + assert [%{line: 7, text: "#trailing 3"}] = three_meta[:trailing_comments] + assert [%{line: 8, text: "# trailing outside"}] = tuple_meta[:trailing_comments] + end + + test "merges comments in 2-tuple" do + quoted = + parse_string!(""" + { + #leading 1 + 1, + #leading 2 + 2 + #trailing 2 + } # trailing outside + """) + + assert {:__block__, tuple_meta, + [ + { + {:__block__, one_meta, [1]}, + {:__block__, two_meta, [2]} + } + ]} = quoted + + assert [%{line: 2, text: "#leading 1"}] = one_meta[:leading_comments] + assert [%{line: 4, text: "#leading 2"}] = two_meta[:leading_comments] + assert [%{line: 6, text: "#trailing 2"}] = two_meta[:trailing_comments] + assert [%{line: 7, text: "# trailing outside"}] = tuple_meta[:trailing_comments] + end + + test "merges inner comments in empty tuple" do + quoted = + parse_string!(""" + { + # inner 1 + # inner 2 + } # trailing outside + """) + + assert {:{}, tuple_meta, []} = quoted + + assert [%{line: 2, text: "# inner 1"}, %{line: 3, text: "# inner 2"}] = + tuple_meta[:inner_comments] + + assert [%{line: 4, text: "# trailing outside"}] = tuple_meta[:trailing_comments] + end + + # Maps + + test "merges comments in maps" do + quoted = + parse_string!(""" + %{ + #leading 1 + 1 => 1, + #leading 2 + 2 => 2, + 3 => 3 + #trailing 3 + } # trailing outside + """) + + assert {:%{}, map_meta, + [ + { + {:__block__, one_key_meta, [1]}, + {:__block__, _, [1]} + }, + { + {:__block__, two_key_meta, [2]}, + {:__block__, _, [2]} + }, + { + {:__block__, _, [3]}, + {:__block__, three_value_meta, [3]} + } + ]} = quoted + + assert [%{line: 2, text: "#leading 1"}] = one_key_meta[:leading_comments] + assert [%{line: 4, text: "#leading 2"}] = two_key_meta[:leading_comments] + assert [%{line: 7, text: "#trailing 3"}] = three_value_meta[:trailing_comments] + assert [%{line: 8, text: "# trailing outside"}] = map_meta[:trailing_comments] + end + + test "merges inner comments in empty maps" do + quoted = + parse_string!(""" + %{ + # inner 1 + # inner 2 + } # trailing outside + """) + + assert {:%{}, map_meta, []} = quoted + + assert [%{line: 2, text: "# inner 1"}, %{line: 3, text: "# inner 2"}] = + map_meta[:inner_comments] + + assert [%{line: 4, text: "# trailing outside"}] = map_meta[:trailing_comments] + end + + test "handles the presence of unquote_splicing" do + quoted = + parse_string!(""" + %{ + # leading baz + :baz => :bat, + :quux => :quuz, + # leading unquote splicing + unquote_splicing(foo: :bar) + # trailing unquote splicing + } + """) + + assert {:%{}, _, + [ + {{:__block__, baz_key_meta, [:baz]}, {:__block__, _, [:bat]}}, + {{:__block__, _, [:quux]}, {:__block__, _, [:quuz]}}, + {:unquote_splicing, unquote_splicing_meta, _} + ]} = quoted + + assert [%{line: 2, text: "# leading baz"}] = baz_key_meta[:leading_comments] + + assert [%{line: 5, text: "# leading unquote splicing"}] = + unquote_splicing_meta[:leading_comments] + + assert [%{line: 7, text: "# trailing unquote splicing"}] = + unquote_splicing_meta[:trailing_comments] + end + + # Structs + + test "merges comments in structs" do + quoted = + parse_string!(""" + %SomeStruct{ + #leading 1 + a: 1, + #leading 2 + b: 2, + c: 3 + #trailing 3 + } # trailing outside + """) + + assert {:%, struct_meta, + [ + {:__aliases__, _, [:SomeStruct]}, + {:%{}, _, + [ + {{:__block__, a_key_meta, [:a]}, {:__block__, _, [1]}}, + {{:__block__, b_key_meta, [:b]}, {:__block__, _, [2]}}, + {{:__block__, _, [:c]}, {:__block__, c_value_meta, [3]}} + ]} + ]} = quoted + + assert [%{line: 2, text: "#leading 1"}] = a_key_meta[:leading_comments] + assert [%{line: 4, text: "#leading 2"}] = b_key_meta[:leading_comments] + assert [%{line: 7, text: "#trailing 3"}] = c_value_meta[:trailing_comments] + assert [%{line: 8, text: "# trailing outside"}] = struct_meta[:trailing_comments] + end + + test "merges inner comments in structs" do + quoted = + parse_string!(""" + %SomeStruct{ + # inner 1 + # inner 2 + } # trailing outside + """) + + assert {:%, struct_meta, + [ + {:__aliases__, _, [:SomeStruct]}, + {:%{}, args_meta, []} + ]} = quoted + + assert [%{line: 2, text: "# inner 1"}, %{line: 3, text: "# inner 2"}] = + args_meta[:inner_comments] + + assert [%{line: 4, text: "# trailing outside"}] = struct_meta[:trailing_comments] + end + + # Stabs (->) + + test "merges comments in anonymous function" do + quoted = + parse_string!(""" + fn -> + # comment + hello + world + # trailing world + end # trailing + """) + + assert {:fn, fn_meta, + [ + {:->, _, + [ + [], + {:__block__, _, [{:hello, hello_meta, _}, {:world, world_meta, _}]} + ]} + ]} = quoted + + assert [%{line: 2, text: "# comment"}] = hello_meta[:leading_comments] + assert [%{line: 5, text: "# trailing world"}] = world_meta[:trailing_comments] + assert [%{line: 6, text: "# trailing"}] = fn_meta[:trailing_comments] + end + + test "merges inner comments in anonymous function" do + quoted = + parse_string!(""" + fn -> + # inner 1 + # inner 2 + end + """) + + assert {:fn, _, + [ + {:->, _, + [ + [], + {:__block__, args_meta, [nil]} + ]} + ]} = quoted + + assert [%{line: 2, text: "# inner 1"}, %{line: 3, text: "# inner 2"}] = + args_meta[:inner_comments] + end + + test "merges trailing comments for do/end stags" do + quoted = + parse_string!(""" + case foo do + _ -> + bar + # trailing + end + """) + + assert {:case, _, + [ + {:foo, _, _}, + [ + {{:__block__, _, [:do]}, [{:->, _, [[{:_, _, _}], {:bar, bar_meta, _}]}]} + ] + ]} = quoted + + assert [%{line: 4, text: "# trailing"}] = bar_meta[:trailing_comments] + end + + test "merges inner comments for do/end stabs" do + quoted = + parse_string!(""" + case foo do + _ -> + # inner + end + """) + + assert {:case, _, + [ + {:foo, _, _}, + [ + {{:__block__, _, [:do]}, + [{:->, _, [[{:_, _, _}], {:__block__, args_meta, [nil]}]}]} + ] + ]} = quoted + + assert [%{line: 3, text: "# inner"}] = args_meta[:inner_comments] + end + + test "merges leading and trailing comments for stabs" do + quoted = + parse_string!(""" + # fn + fn + # before head + # middle head + hello -> + # after head + # before body + # middle body + world + # after body + end + """) + + assert {:fn, fn_meta, + [ + {:->, _, + [ + [{:hello, hello_meta, _}], + {:world, world_meta, _} + ]} + ]} = quoted + + assert [%{line: 1, text: "# fn"}] = fn_meta[:leading_comments] + + assert [%{line: 3, text: "# before head"}, %{line: 4, text: "# middle head"}] = + hello_meta[:leading_comments] + + assert [ + %{line: 6, text: "# after head"}, + %{line: 7, text: "# before body"}, + %{line: 8, text: "# middle body"} + ] = world_meta[:leading_comments] + + assert [%{line: 10, text: "# after body"}] = world_meta[:trailing_comments] + end + end +end diff --git a/lib/elixir/test/elixir/code_formatter/ast_comments_test.exs b/lib/elixir/test/elixir/code_formatter/ast_comments_test.exs new file mode 100644 index 0000000000..576f6fab83 --- /dev/null +++ b/lib/elixir/test/elixir/code_formatter/ast_comments_test.exs @@ -0,0 +1,521 @@ +Code.require_file("../test_helper.exs", __DIR__) + +defmodule Code.Formatter.AstCommentsTest do + use ExUnit.Case, async: true + + def parse_string!(string) do + Code.string_to_quoted!(string, include_comments: true) + end + + describe "merge_comments/2" do + test "merges comments in empty AST" do + quoted = + parse_string!(""" + # some comment + # another comment + """) + + assert {:__block__, meta, []} = quoted + + assert [%{line: 1, text: "# some comment"}, %{line: 2, text: "# another comment"}] = + meta[:inner_comments] + end + + test "merges leading comments in assorted terms" do + quoted = + parse_string!(""" + # leading var + var + # trailing var + """) + + assert {:var, meta, _} = quoted + + assert [%{line: 1, text: "# leading var"}] = meta[:leading_comments] + assert [%{line: 3, text: "# trailing var"}] = meta[:trailing_comments] + + quoted = + parse_string!(""" + # leading 1 + 1 + # trailing 1 + """) + + assert {:__block__, one_meta, [1]} = quoted + + assert [%{line: 1, text: "# leading 1"}] = one_meta[:leading_comments] + assert [%{line: 3, text: "# trailing 1"}] = one_meta[:trailing_comments] + + quoted = + parse_string!(""" + # leading qualified call + Foo.bar(baz) + # trailing qualified call + """) + + assert {{:., _, [_Foo, _bar]}, meta, _} = quoted + + assert [%{line: 1, text: "# leading qualified call"}] = meta[:leading_comments] + assert [%{line: 3, text: "# trailing qualified call"}] = meta[:trailing_comments] + + quoted = + parse_string!(""" + # leading qualified call + Foo. + # leading bar + bar(baz) + # trailing qualified call + """) + + assert {{:., _, [_Foo, _]}, meta, + [ + {:baz, _, _} + ]} = quoted + + assert [%{line: 1, text: "# leading qualified call"}, %{line: 3, text: "# leading bar"}] = + meta[:leading_comments] + + assert [%{line: 5, text: "# trailing qualified call"}] = meta[:trailing_comments] + end + + # Do/end blocks + + test "merges comments in do/end block" do + quoted = + parse_string!(""" + def a do + foo() + :ok + # A + end # B + """) + + assert {:def, def_meta, + [ + {:a, _, _}, + [ + {{:__block__, _, [:do]}, + {:__block__, _, + [ + {:foo, _, _}, + {:__block__, meta, [:ok]} + ]}} + ] + ]} = + quoted + + assert [%{line: 4, text: "# A"}] = meta[:trailing_comments] + + assert [%{line: 5, text: "# B"}] = def_meta[:trailing_comments] + end + + test "merges comments for named do/end blocks" do + quoted = + parse_string!(""" + def a do + # leading var1 + var1 + # trailing var1 + else + # leading var2 + var2 + # trailing var2 + catch + # leading var3 + var3 + # trailing var3 + rescue + # leading var4 + var4 + # trailing var4 + after + # leading var5 + var5 + # trailing var5 + end + """) + + assert {:def, _, + [ + {:a, _, _}, + [ + {{:__block__, _, [:do]}, {:var1, var1_meta, _}}, + {{:__block__, _, [:else]}, {:var2, var2_meta, _}}, + {{:__block__, _, [:catch]}, {:var3, var3_meta, _}}, + {{:__block__, _, [:rescue]}, {:var4, var4_meta, _}}, + {{:__block__, _, [:after]}, {:var5, var5_meta, _}} + ] + ]} = + quoted + + assert [%{line: 2, text: "# leading var1"}] = var1_meta[:leading_comments] + assert [%{line: 4, text: "# trailing var1"}] = var1_meta[:trailing_comments] + assert [%{line: 6, text: "# leading var2"}] = var2_meta[:leading_comments] + assert [%{line: 8, text: "# trailing var2"}] = var2_meta[:trailing_comments] + assert [%{line: 10, text: "# leading var3"}] = var3_meta[:leading_comments] + assert [%{line: 12, text: "# trailing var3"}] = var3_meta[:trailing_comments] + assert [%{line: 14, text: "# leading var4"}] = var4_meta[:leading_comments] + assert [%{line: 16, text: "# trailing var4"}] = var4_meta[:trailing_comments] + assert [%{line: 18, text: "# leading var5"}] = var5_meta[:leading_comments] + assert [%{line: 20, text: "# trailing var5"}] = var5_meta[:trailing_comments] + end + + test "merges inner comments for empty named do/end blocks" do + quoted = + parse_string!(""" + def a do + # inside do + else + # inside else + catch + # inside catch + rescue + # inside rescue + after + # inside after + end + """) + + assert {:def, _, + [ + {:a, _, _}, + [ + {{:__block__, _, [:do]}, {:__block__, do_meta, _}}, + {{:__block__, _, [:else]}, {:__block__, else_meta, _}}, + {{:__block__, _, [:catch]}, {:__block__, catch_meta, _}}, + {{:__block__, _, [:rescue]}, {:__block__, rescue_meta, _}}, + {{:__block__, _, [:after]}, {:__block__, after_meta, _}} + ] + ]} = + quoted + + assert [%{line: 2, text: "# inside do"}] = do_meta[:inner_comments] + assert [%{line: 4, text: "# inside else"}] = else_meta[:inner_comments] + assert [%{line: 6, text: "# inside catch"}] = catch_meta[:inner_comments] + assert [%{line: 8, text: "# inside rescue"}] = rescue_meta[:inner_comments] + assert [%{line: 10, text: "# inside after"}] = after_meta[:inner_comments] + end + + # Lists + + test "merges comments in list" do + quoted = + parse_string!(""" + [ + #leading 1 + 1, + #leading 2 + 2, + 3 + #trailing 3 + ] # trailing outside + """) + + assert {:__block__, list_meta, + [ + [ + {:__block__, one_meta, [1]}, + {:__block__, two_meta, [2]}, + {:__block__, three_meta, [3]} + ] + ]} = quoted + + assert [%{line: 2, text: "#leading 1"}] = one_meta[:leading_comments] + assert [%{line: 4, text: "#leading 2"}] = two_meta[:leading_comments] + assert [%{line: 7, text: "#trailing 3"}] = three_meta[:trailing_comments] + assert [%{line: 8, text: "# trailing outside"}] = list_meta[:trailing_comments] + end + + test "merges inner comments in empty list" do + quoted = + parse_string!(""" + [ + # inner 1 + # inner 2 + ] # trailing outside + """) + + assert {:__block__, list_meta, [[]]} = quoted + + assert [%{line: 2, text: "# inner 1"}, %{line: 3, text: "# inner 2"}] = + list_meta[:inner_comments] + + assert [%{line: 4, text: "# trailing outside"}] = list_meta[:trailing_comments] + end + + # Keyword lists + + test "merges comments in keyword list" do + quoted = + parse_string!(""" + [ + #leading a + a: 1, + #leading b + b: 2, + c: 3 + #trailing 3 + ] # trailing outside + """) + + assert {:__block__, keyword_list_meta, + [ + [ + { + {:__block__, a_key_meta, [:a]}, + {:__block__, _, [1]} + }, + { + {:__block__, b_key_meta, [:b]}, + {:__block__, _, [2]} + }, + { + {:__block__, _, [:c]}, + {:__block__, c_value_meta, [3]} + } + ] + ]} = quoted + + assert [%{line: 2, text: "#leading a"}] = a_key_meta[:leading_comments] + assert [%{line: 4, text: "#leading b"}] = b_key_meta[:leading_comments] + assert [%{line: 7, text: "#trailing 3"}] = c_value_meta[:trailing_comments] + assert [%{line: 8, text: "# trailing outside"}] = keyword_list_meta[:trailing_comments] + end + + test "merges comments in partial keyword list" do + quoted = + parse_string!(""" + [ + #leading 1 + 1, + #leading b + b: 2 + #trailing b + ] # trailing outside + """) + + assert {:__block__, keyword_list_meta, + [ + [ + {:__block__, one_key_meta, [1]}, + { + {:__block__, b_key_meta, [:b]}, + {:__block__, b_value_meta, [2]} + } + ] + ]} = quoted + + assert [%{line: 2, text: "#leading 1"}] = one_key_meta[:leading_comments] + assert [%{line: 4, text: "#leading b"}] = b_key_meta[:leading_comments] + assert [%{line: 6, text: "#trailing b"}] = b_value_meta[:trailing_comments] + assert [%{line: 7, text: "# trailing outside"}] = keyword_list_meta[:trailing_comments] + end + + # Tuples + + test "merges comments in n-tuple" do + quoted = + parse_string!(""" + { + #leading 1 + 1, + #leading 2 + 2, + 3 + #trailing 3 + } # trailing outside + """) + + assert {:{}, tuple_meta, + [ + {:__block__, one_meta, [1]}, + {:__block__, two_meta, [2]}, + {:__block__, three_meta, [3]} + ]} = quoted + + assert [%{line: 2, text: "#leading 1"}] = one_meta[:leading_comments] + assert [%{line: 4, text: "#leading 2"}] = two_meta[:leading_comments] + assert [%{line: 7, text: "#trailing 3"}] = three_meta[:trailing_comments] + assert [%{line: 8, text: "# trailing outside"}] = tuple_meta[:trailing_comments] + end + + test "merges comments in 2-tuple" do + quoted = + parse_string!(""" + { + #leading 1 + 1, + #leading 2 + 2 + #trailing 2 + } # trailing outside + """) + + assert {:__block__, tuple_meta, + [ + { + {:__block__, one_meta, [1]}, + {:__block__, two_meta, [2]} + } + ]} = quoted + + assert [%{line: 2, text: "#leading 1"}] = one_meta[:leading_comments] + assert [%{line: 4, text: "#leading 2"}] = two_meta[:leading_comments] + assert [%{line: 6, text: "#trailing 2"}] = two_meta[:trailing_comments] + assert [%{line: 7, text: "# trailing outside"}] = tuple_meta[:trailing_comments] + end + + test "merges inner comments in empty tuple" do + quoted = + parse_string!(""" + { + # inner 1 + # inner 2 + } # trailing outside + """) + + assert {:{}, tuple_meta, []} = quoted + + assert [%{line: 2, text: "# inner 1"}, %{line: 3, text: "# inner 2"}] = + tuple_meta[:inner_comments] + + assert [%{line: 4, text: "# trailing outside"}] = tuple_meta[:trailing_comments] + end + + # Maps + + test "merges comments in maps" do + quoted = + parse_string!(""" + %{ + #leading 1 + 1 => 1, + #leading 2 + 2 => 2, + 3 => 3 + #trailing 3 + } # trailing outside + """) + + assert {:%{}, map_meta, + [ + { + {:__block__, one_key_meta, [1]}, + {:__block__, _, [1]} + }, + { + {:__block__, two_key_meta, [2]}, + {:__block__, _, [2]} + }, + { + {:__block__, _, [3]}, + {:__block__, three_value_meta, [3]} + } + ]} = quoted + + assert [%{line: 2, text: "#leading 1"}] = one_key_meta[:leading_comments] + assert [%{line: 4, text: "#leading 2"}] = two_key_meta[:leading_comments] + assert [%{line: 7, text: "#trailing 3"}] = three_value_meta[:trailing_comments] + assert [%{line: 8, text: "# trailing outside"}] = map_meta[:trailing_comments] + end + + test "merges inner comments in empty maps" do + quoted = + parse_string!(""" + %{ + # inner 1 + # inner 2 + } # trailing outside + """) + + assert {:%{}, map_meta, []} = quoted + + assert [%{line: 2, text: "# inner 1"}, %{line: 3, text: "# inner 2"}] = + map_meta[:inner_comments] + + assert [%{line: 4, text: "# trailing outside"}] = map_meta[:trailing_comments] + end + end + + test "handles the presence of unquote_splicing" do + quoted = + parse_string!(""" + %{ + # leading baz + :baz => :bat, + :quux => :quuz, + # leading unquote splicing + unquote_splicing(foo: :bar) + # trailing unquote splicing + } + """) + + assert {:%{}, _, + [ + {{:__block__, baz_key_meta, [:baz]}, {:__block__, _, [:bat]}}, + {{:__block__, _, [:quux]}, {:__block__, _, [:quuz]}}, + {:unquote_splicing, unquote_splicing_meta, _} + ]} = quoted + + assert [%{line: 2, text: "# leading baz"}] = baz_key_meta[:leading_comments] + + assert [%{line: 5, text: "# leading unquote splicing"}] = + unquote_splicing_meta[:leading_comments] + + assert [%{line: 7, text: "# trailing unquote splicing"}] = + unquote_splicing_meta[:trailing_comments] + end + + # Structs + + test "merges comments in structs" do + quoted = + parse_string!(""" + %SomeStruct{ + #leading 1 + a: 1, + #leading 2 + b: 2, + c: 3 + #trailing 3 + } # trailing outside + """) + + assert {:%, struct_meta, + [ + {:__aliases__, _, [:SomeStruct]}, + {:%{}, _, + [ + {{:__block__, a_key_meta, [:a]}, {:__block__, _, [1]}}, + {{:__block__, b_key_meta, [:b]}, {:__block__, _, [2]}}, + {{:__block__, _, [:c]}, {:__block__, c_value_meta, [3]}} + ]} + ]} = quoted + + assert [%{line: 2, text: "#leading 1"}] = a_key_meta[:leading_comments] + assert [%{line: 4, text: "#leading 2"}] = b_key_meta[:leading_comments] + assert [%{line: 7, text: "#trailing 3"}] = c_value_meta[:trailing_comments] + assert [%{line: 8, text: "# trailing outside"}] = struct_meta[:trailing_comments] + end + + test "merges inner comments in structs" do + quoted = + parse_string!(""" + %SomeStruct{ + # inner 1 + # inner 2 + } # trailing outside + """) + + assert {:%, struct_meta, + [ + {:__aliases__, _, [:SomeStruct]}, + {:%{}, args_meta, []} + ]} = quoted + + assert [%{line: 2, text: "# inner 1"}, %{line: 3, text: "# inner 2"}] = + args_meta[:inner_comments] + + assert [%{line: 4, text: "# trailing outside"}] = struct_meta[:trailing_comments] + end +end From e3e4fac770213e91b549d5342741a126af79b3a2 Mon Sep 17 00:00:00 2001 From: doorgan Date: Fri, 21 Mar 2025 14:50:49 -0300 Subject: [PATCH 2/7] Update formatter to look for AST comments --- lib/elixir/lib/code/formatter.ex | 121 +++++++++++++++++-------------- 1 file changed, 68 insertions(+), 53 deletions(-) diff --git a/lib/elixir/lib/code/formatter.ex b/lib/elixir/lib/code/formatter.ex index 7e1ea6ca87..80124657bf 100644 --- a/lib/elixir/lib/code/formatter.ex +++ b/lib/elixir/lib/code/formatter.ex @@ -641,8 +641,21 @@ defmodule Code.Formatter do paren_fun_to_algebra(paren_fun, min_line, max_line, state) end - defp block_to_algebra({:__block__, _, []}, min_line, max_line, state) do - block_args_to_algebra([], min_line, max_line, state) + defp block_to_algebra({:__block__, meta, args}, _min_line, _max_line, state) when args in [[], [nil]] do + inner_comments = meta[:inner_comments] || [] + comments_docs = + Enum.map(inner_comments, fn comment -> + comment = format_comment(comment) + {comment.text, @empty, 1} + end) + + docs = merge_algebra_with_comments(comments_docs, @empty) + + case docs do + [] -> {@empty, state} + [line] -> {line, state} + lines -> {lines |> Enum.reduce(&line(&2, &1)) |> force_unfit(), state} + end end defp block_to_algebra({:__block__, _, [_, _ | _] = args}, min_line, max_line, state) do @@ -1827,7 +1840,12 @@ defmodule Code.Formatter do end {args_docs, comments?, state} = - quoted_to_algebra_with_comments(args, acc, min_line, max_line, state, arg_to_algebra) + case args do + [] -> + quoted_to_algebra_with_comments(args, acc, min_line, max_line, state, arg_to_algebra) + _ -> + quoted_to_algebra_with_comments(args, acc, min_line, max_line, state, arg_to_algebra) + end cond do args_docs == [] -> @@ -2089,69 +2107,67 @@ defmodule Code.Formatter do end ## Quoted helpers for comments - - defp quoted_to_algebra_with_comments(args, acc, min_line, max_line, state, fun) do - {pre_comments, state} = - get_and_update_in(state.comments, fn comments -> - Enum.split_while(comments, fn %{line: line} -> line <= min_line end) - end) - - {reverse_docs, comments?, state} = - if state.comments == [] do - each_quoted_to_algebra_without_comments(args, acc, state, fun) - else - each_quoted_to_algebra_with_comments(args, acc, max_line, state, false, fun) - end + defp quoted_to_algebra_with_comments(args, acc, _min_line, _max_line, state, fun) do + {reverse_docs, comments?, state} = each_quoted_to_algebra_with_comments(args, acc, state, fun, false) docs = merge_algebra_with_comments(Enum.reverse(reverse_docs), @empty) - {docs, comments?, update_in(state.comments, &(pre_comments ++ &1))} + + {docs, comments?, state} end - defp each_quoted_to_algebra_without_comments([], acc, state, _fun) do - {acc, false, state} + defp each_quoted_to_algebra_with_comments([], acc, state, _fun, comments?) do + {acc, comments?, state} end - defp each_quoted_to_algebra_without_comments([arg | args], acc, state, fun) do + defp each_quoted_to_algebra_with_comments([arg | args], acc, state, fun, comments?) do {doc_triplet, state} = fun.(arg, args, state) - acc = [doc_triplet | acc] - each_quoted_to_algebra_without_comments(args, acc, state, fun) - end - defp each_quoted_to_algebra_with_comments([], acc, max_line, state, comments?, _fun) do - {acc, comments, comments?} = extract_comments_before(max_line, acc, state.comments, comments?) - {acc, comments?, %{state | comments: comments}} - end - defp each_quoted_to_algebra_with_comments([arg | args], acc, max_line, state, comments?, fun) do case traverse_line(arg, {@max_line, @min_line}) do {@max_line, @min_line} -> - {doc_triplet, state} = fun.(arg, args, state) acc = [doc_triplet | acc] - each_quoted_to_algebra_with_comments(args, acc, max_line, state, comments?, fun) + each_quoted_to_algebra_with_comments(args, acc, state, fun, comments?) {doc_start, doc_end} -> - {acc, comments, comments?} = - extract_comments_before(doc_start, acc, state.comments, comments?) + {leading_comments, trailing_comments} = + case arg do + {_, meta, _} -> + leading_comments = meta[:leading_comments] || [] + trailing_comments = meta[:trailing_comments] || [] + {leading_comments, trailing_comments} + + {{_, left_meta, _}, {_, right_meta, _}} -> + leading_comments = left_meta[:leading_comments] || [] + trailing_comments = right_meta[:trailing_comments] || [] + + {leading_comments, trailing_comments} + _ -> + {[], []} + end - {doc_triplet, state} = fun.(arg, args, %{state | comments: comments}) + comments? = leading_comments != [] or trailing_comments != [] - {acc, comments, comments?} = - extract_comments_trailing(doc_start, doc_end, acc, state.comments, comments?) + leading_docs = build_leading_comments([], leading_comments, doc_start) + trailing_docs = build_trailing_comments([], trailing_comments) - acc = [adjust_trailing_newlines(doc_triplet, doc_end, comments) | acc] - state = %{state | comments: comments} - each_quoted_to_algebra_with_comments(args, acc, max_line, state, comments?, fun) + doc_triplet = adjust_trailing_newlines(doc_triplet, doc_end, trailing_comments) + + acc = Enum.concat([trailing_docs, [doc_triplet], leading_docs, acc]) + + each_quoted_to_algebra_with_comments(args, acc, state, fun, comments?) end end - defp extract_comments_before(max, acc, [%{line: line} = comment | rest], _) when line < max do - %{previous_eol_count: previous, next_eol_count: next, text: doc} = comment - acc = [{doc, @empty, next} | add_previous_to_acc(acc, previous)] - extract_comments_before(max, acc, rest, true) - end + defp build_leading_comments(acc, [], _), do: acc - defp extract_comments_before(_max, acc, rest, comments?) do - {acc, rest, comments?} + defp build_leading_comments(acc, [comment | rest], doc_start) do + comment = format_comment(comment) + %{previous_eol_count: previous, next_eol_count: next, text: doc, line: line} = comment + # If the comment is on the same line as the document, we need to adjust the newlines + # such that the comment is placed right above the document line. + next = if line == doc_start, do: 1, else: next + acc = [{doc, @empty, next} | add_previous_to_acc(acc, previous)] + build_leading_comments(acc, rest, doc_start) end defp add_previous_to_acc([{doc, next_line, newlines} | acc], previous) when newlines < previous, @@ -2159,15 +2175,13 @@ defmodule Code.Formatter do defp add_previous_to_acc(acc, _previous), do: acc + defp build_trailing_comments(acc, []), do: acc - defp extract_comments_trailing(min, max, acc, [%{line: line, text: doc_comment} | rest], _) - when line >= min and line <= max do - acc = [{doc_comment, @empty, 1} | acc] - extract_comments_trailing(min, max, acc, rest, true) - end - - defp extract_comments_trailing(_min, _max, acc, rest, comments?) do - {acc, rest, comments?} + defp build_trailing_comments(acc, [comment | rest]) do + comment = format_comment(comment) + %{previous_eol_count: previous, next_eol_count: next, text: doc} = comment + acc = [{doc, @empty, next} | acc] + build_trailing_comments(acc, rest) end # If the document is immediately followed by comment which is followed by newlines, @@ -2179,6 +2193,7 @@ defmodule Code.Formatter do defp adjust_trailing_newlines(doc_triplet, _, _), do: doc_triplet + defp traverse_line({expr, meta, args}, {min, max}) do # This is a hot path, so use :lists.keyfind/3 instead Keyword.fetch!/2 acc = From c9e9a0206f4cfb68e07a188989c2e6dc26586cc9 Mon Sep 17 00:00:00 2001 From: doorgan Date: Fri, 28 Mar 2025 14:46:05 -0300 Subject: [PATCH 3/7] Handle more scenarios and update formatter --- lib/elixir/lib/code.ex | 97 ++- lib/elixir/lib/code/comments.ex | 658 ++++++++++++++---- lib/elixir/lib/code/formatter.ex | 384 +++++++--- .../test/elixir/code/ast_comments_test.exs | 26 +- .../code_formatter/ast_comments_test.exs | 521 -------------- 5 files changed, 898 insertions(+), 788 deletions(-) delete mode 100644 lib/elixir/test/elixir/code_formatter/ast_comments_test.exs diff --git a/lib/elixir/lib/code.ex b/lib/elixir/lib/code.ex index 8665e3da75..5eb4d41e56 100644 --- a/lib/elixir/lib/code.ex +++ b/lib/elixir/lib/code.ex @@ -1209,6 +1209,12 @@ defmodule Code do * `:emit_warnings` (since v1.16.0) - when `false`, does not emit tokenizing/parsing related warnings. Defaults to `true`. + * `:include_comments` (since v1.19.0) - when `true`, includes comments + in the quoted form. Defaults to `false`. If this option is set to + `true`, the `:literal_encoder` option must be set to a function + that ensures all literals are annotated, for example + `&{:ok, {:__block__, &2, [&1]}}`. + ## `Macro.to_string/2` The opposite of converting a string to its quoted form is @@ -1248,6 +1254,64 @@ defmodule Code do * atoms used to represent single-letter sigils like `:sigil_X` (but multi-letter sigils like `:sigil_XYZ` are encoded). + ## Comments + + When `include_comments: true` is passed, comments are included in the + quoted form. + + There are three types of comments: + - `:leading_comments`: Comments that are located before a node, + or in the same line. + + Examples: + + # This is a leading comment + foo # This one too + + - `:trailing_comments`: Comments that are located after a node, and + before the end of the parent enclosing the node(or the root document). + + Examples: + + foo + # This is a trailing comment + # This one too + + - `:inner_comments`: Comments that are located inside an empty node. + + Examples: + + foo do + # This is an inner comment + end + + [ + # This is an inner comment + ] + + %{ + # This is an inner comment + } + + A comment may be considered inner or trailing depending on wether the enclosing + node is empty or not. For example, in the following code: + + foo do + # This is an inner comment + end + + The comment is considered inner because the `do` block is empty. However, in the + following code: + + foo do + bar + # This is a trailing comment + end + + The comment is considered trailing to `bar` because the `do` block is not empty. + + In the case no nodes are present in the AST but there are comments, they are + inserted into a placeholder `:__block__` node as `:inner_comments`. """ @spec string_to_quoted(List.Chars.t(), keyword) :: {:ok, Macro.t()} | {:error, {location :: keyword, binary | {binary, binary}, binary}} @@ -1255,14 +1319,22 @@ defmodule Code do file = Keyword.get(opts, :file, "nofile") line = Keyword.get(opts, :line, 1) column = Keyword.get(opts, :column, 1) - include_comments = Keyword.get(opts, :include_comments, false) + include_comments? = Keyword.get(opts, :include_comments, false) Process.put(:code_formatter_comments, []) - opts = [preserve_comments: &preserve_comments/5] ++ opts + opts = + if include_comments? do + [ + preserve_comments: &preserve_comments/5, + token_metadata: true, + ] ++ opts + else + opts + end with {:ok, tokens} <- :elixir.string_to_tokens(to_charlist(string), line, column, file, opts), {:ok, quoted} <- :elixir.tokens_to_quoted(tokens, file, opts) do - if include_comments do + if include_comments? do quoted = Code.Normalizer.normalize(quoted) quoted = Code.Comments.merge_comments(quoted, Process.get(:code_formatter_comments)) @@ -1292,26 +1364,23 @@ defmodule Code do file = Keyword.get(opts, :file, "nofile") line = Keyword.get(opts, :line, 1) column = Keyword.get(opts, :column, 1) - include_comments = Keyword.get(opts, :include_comments, false) + include_comments? = Keyword.get(opts, :include_comments, false) Process.put(:code_formatter_comments, []) - opts = - if include_comments do - [preserve_comments: &preserve_comments/5, - literal_encoder: &{:ok, {:__block__, &2, [&1]}}, - token_metadata: true, - unescape: false, - columns: true, -] ++ opts + opts = + if include_comments? do + [ + preserve_comments: &preserve_comments/5, + token_metadata: true, + ] ++ opts else opts end quoted = :elixir.string_to_quoted!(to_charlist(string), line, column, file, opts) - if include_comments do - # quoted = Code.Normalizer.normalize(quoted) + if include_comments? do Code.Comments.merge_comments(quoted, Process.get(:code_formatter_comments)) else quoted diff --git a/lib/elixir/lib/code/comments.ex b/lib/elixir/lib/code/comments.ex index d97e05b21b..9ce3eb8839 100644 --- a/lib/elixir/lib/code/comments.ex +++ b/lib/elixir/lib/code/comments.ex @@ -9,72 +9,22 @@ defmodule Code.Comments do @doc """ Merges the comments into the given quoted expression. - - There are three types of comments: - - `:leading_comments`: Comments that are located before a node, - or in the same line. - - Examples: - - # This is a leading comment - foo # This one too - - - `:trailing_comments`: Comments that are located after a node, and - before the end of the parent enclosing the node(or the root document). - - Examples: - - foo - # This is a trailing comment - # This one too - - - `:inner_comments`: Comments that are located inside an empty node. - - Examples: - - foo do - # This is an inner comment - end - - [ - # This is an inner comment - ] - - %{ - # This is an inner comment - } - - A comment may be considered inner or trailing depending on wether the enclosing - node is empty or not. For example, in the following code: - - foo do - # This is an inner comment - end - - The comment is considered inner because the `do` block is empty. However, in the - following code: - - foo do - bar - # This is a trailing comment - end - - The comment is considered trailing to `bar` because the `do` block is not empty. - - In the case no nodes are present in the AST but there are comments, they are - inserted into a placeholder `:__block__` node as `:inner_comments`. """ @spec merge_comments(Macro.t(), list(map)) :: Macro.t() def merge_comments({:__block__, _, []} = empty_ast, comments) do comments = Enum.sort_by(comments, & &1.line) - put_comments(empty_ast, :inner_comments, comments) + + empty_ast + |> ensure_comments_meta() + |> put_comments(:inner_comments, comments) end + def merge_comments(quoted, comments) do comments = Enum.sort_by(comments, & &1.line) state = %{ comments: comments, - parent_doend_meta: [] + parent_meta: [] } {quoted, %{comments: leftovers}} = Macro.prewalk(quoted, state, &do_merge_comments/2) @@ -88,6 +38,7 @@ defmodule Code.Comments do case last_arg do nil -> append_comments(quoted, :inner_comments, comments) + {_, _, _} = last_arg -> last_arg = append_comments(last_arg, :trailing_comments, comments) @@ -104,7 +55,8 @@ defmodule Code.Comments do end defp do_merge_comments({_, _, _} = quoted, state) do - {quoted, state} = merge_trailing_comments(quoted, state) + quoted = ensure_comments_meta(quoted) + {quoted, state} = merge_mixed_comments(quoted, state) merge_leading_comments(quoted, state) end @@ -112,19 +64,26 @@ defmodule Code.Comments do {quoted, state} end + defp ensure_comments_meta({form, meta, args}) do + meta = + meta + |> Keyword.put_new(:leading_comments, []) + |> Keyword.put_new(:trailing_comments, []) + |> Keyword.put_new(:inner_comments, []) + + {form, meta, args} + end + defp merge_leading_comments(quoted, state) do # If a comment is placed on top of a pipeline or binary operator line, # we should not merge it with the operator itself. Instead, we should # merge it with the first argument of the pipeline. - # - # This avoids the comment being moved up when formatting the code. with {form, _, _} <- quoted, - false <- is_arrow_op(form), - :error <- Code.Identifier.binary_op(form) do + false <- is_arrow_op(form) do {comments, rest} = gather_leading_comments_for_node(quoted, state.comments) comments = Enum.sort_by(comments, & &1.line) - quoted = put_comments(quoted, :leading_comments, comments) + quoted = put_leading_comments(quoted, comments) {quoted, %{state | comments: rest}} else _ -> @@ -145,72 +104,184 @@ defmodule Code.Comments do end end) + rest = Enum.reverse(rest) + {comments, rest} end + defp put_leading_comments({form, meta, args}, comments) do + with {_, _} <- Code.Identifier.binary_op(form), + [_, _] <- args do + put_binary_op_comments({form, meta, args}, comments) + else + _ -> + append_comments({form, meta, args}, :leading_comments, comments) + end + end + + defp put_trailing_comments({form, meta, args}, comments) do + with {_, _} <- Code.Identifier.binary_op(form), + [{_, _, _} = left, {_, _, _} = right] <- args do + right = append_comments(right, :trailing_comments, comments) + + {form, meta, [left, right]} + else + _ -> + append_comments({form, meta, args}, :trailing_comments, comments) + end + end + + defp put_binary_op_comments({_, _, [left, right]} = binary_op, comments) do + {leading_comments, rest} = + Enum.split_with(comments, &(&1.line <= get_line(left))) + + right_node = + case right do + [{_, _, _} = first | _other] -> + first + + [{_, right} | _other] -> + right + + _ -> + right + end + + {trailing_comments, rest} = + Enum.split_with( + rest, + &(&1.line > get_line(right_node) && &1.line < get_end_line(right, get_line(right_node))) + ) + + {op_leading_comments, _rest} = + Enum.split_with(rest, &(&1.line <= get_line(binary_op))) + + left = append_comments(left, :leading_comments, leading_comments) + + # It is generally inconvenient to attach comments to the operator itself. + # Considering the following example: + # + # one + # # when two + # # when three + # when four + # # | five + # | six + # + # The AST for the above code will be equivalent to this: + # + # when + # / \ + # one :| + # / \ + # four six + # + # Putting the comments on the operator makes formatting harder to perform, as + # it would need to hoist comments from child nodes above the operator location. + # Having the `# when two` and `# when three` comments as trailing for the left + # node is more convenient for formatting. + # It is also more intuitive, since those comments are related to the left node, + # not the operator itself. + # + # The same applies for the `:|` operator; the `# | five` comment is attached to + # the left node `four`, not the operator. + left = append_comments(left, :trailing_comments, op_leading_comments) + + right = + case right do + [{key, value} | other] -> + value = append_comments(value, :trailing_comments, trailing_comments) + [{key, value} | other] + + [first | other] -> + first = append_comments(first, :trailing_comments, trailing_comments) + [first | other] + + _ -> + append_comments(right, :trailing_comments, trailing_comments) + end + + put_args(binary_op, [left, right]) + end + # Structs - defp merge_trailing_comments({:%, _, [name, args]} = quoted, state) do - {args, comments} = merge_trailing_comments(args, state.comments) + defp merge_mixed_comments({:%, _, [name, args]} = quoted, state) do + {args, state} = merge_mixed_comments(args, state) quoted = put_args(quoted, [name, args]) - {quoted, %{state | comments: comments}} + {quoted, state} end - # Maps - defp merge_trailing_comments({:%{}, _, [{_key, _value} | _] = args} = quoted, %{comments: comments} = state) do - case List.pop_at(args, -1) do - {{last_key, last_value}, args} -> - start_line = get_line(last_value) - end_line = get_end_line(quoted, start_line) + # Map update + defp merge_mixed_comments({:%{}, _, [{:|, pipe_meta, [left, right]}]} = quoted, state) + when is_list(right) do + {right, state} = merge_map_args_trailing_comments(quoted, right, state) - {trailing_comments, comments} = - Enum.split_with(comments, & &1.line > start_line and &1.line < end_line) + quoted = put_args(quoted, [{:|, pipe_meta, [left, right]}]) - last_value = append_comments(last_value, :trailing_comments, trailing_comments) + {quoted, state} + end - args = args ++ [{last_key, last_value}] + # Maps + defp merge_mixed_comments({:%{}, _, [{_key, _value} | _] = args} = quoted, state) do + {args, state} = merge_map_args_trailing_comments(quoted, args, state) - quoted = put_args(quoted, args) - {quoted, %{state | comments: comments}} + quoted = put_args(quoted, args) + {quoted, state} + end - {{:unquote_splicing, _, _} = unquote_splicing, other} -> - start_line = get_line(unquote_splicing) - end_line = get_end_line(quoted, start_line) + # Binary interpolation + defp merge_mixed_comments({:<<>>, _meta, args} = quoted, state) do + if interpolated?(args) do + {args, state} = + Enum.map_reduce(args, state, &merge_interpolation_comments/2) - {trailing_comments, comments} = - Enum.split_with(comments, & &1.line > start_line and &1.line < end_line) + quoted = put_args(quoted, args) - unquote_splicing = append_comments(unquote_splicing, :trailing_comments, trailing_comments) + {quoted, state} + else + merge_call_comments(quoted, state) + end + end - args = other ++ [unquote_splicing] - quoted = put_args(quoted, args) + # List interpolation + defp merge_mixed_comments( + {{:., _dot_meta, [List, :to_charlist]}, _meta, [args]} = quoted, + state + ) do + {args, state} = + Enum.map_reduce(args, state, &merge_interpolation_comments/2) - {quoted, %{state | comments: comments}} - end + quoted = put_args(quoted, [args]) + + {quoted, state} end # Lists - defp merge_trailing_comments({:__block__, _, [args]} = quoted, %{comments: comments} = state) when is_list(args) do + defp merge_mixed_comments({:__block__, _, [args]} = quoted, %{comments: comments} = state) + when is_list(args) do {quoted, comments} = case List.pop_at(args, -1) do {nil, _} -> + # There's no items in the list, merge the comments as inner comments start_line = get_line(quoted) end_line = get_end_line(quoted, start_line) {trailing_comments, comments} = - Enum.split_with(comments, & &1.line > start_line and &1.line < end_line) + Enum.split_with(comments, &(&1.line > start_line and &1.line < end_line)) quoted = append_comments(quoted, :inner_comments, trailing_comments) {quoted, comments} - {{last_key, last_value}, args} -> + {{last_key, last_value}, args} -> + # Partial keyword list, merge the comments as trailing for the value part start_line = get_line(last_value) end_line = get_end_line(quoted, start_line) {trailing_comments, comments} = - Enum.split_with(comments, & &1.line > start_line and &1.line < end_line) + Enum.split_with(comments, &(&1.line > start_line and &1.line < end_line)) last_value = append_comments(last_value, :trailing_comments, trailing_comments) @@ -218,26 +289,34 @@ defmodule Code.Comments do quoted = put_args(quoted, [args]) {quoted, comments} + {{:unquote_splicing, _, _} = unquote_splicing, other} -> start_line = get_line(unquote_splicing) end_line = get_end_line(quoted, start_line) {trailing_comments, comments} = - Enum.split_with(comments, & &1.line > start_line and &1.line < end_line) + Enum.split_with(comments, &(&1.line > start_line and &1.line < end_line)) - unquote_splicing = append_comments(unquote_splicing, :trailing_comments, trailing_comments) + unquote_splicing = + append_comments(unquote_splicing, :trailing_comments, trailing_comments) args = other ++ [unquote_splicing] quoted = put_args(quoted, [args]) {quoted, comments} + {{:__block__, _, [_, _ | _]}, _args} -> + # In the case of a block in the list, there are no comments to merge, + # so we skip it. Otherwise we may attach trailing comments inside the + # block to the block itself, which is not the desired behavior. + {quoted, comments} + {{_, _, _} = value, args} -> start_line = get_line(value) end_line = get_end_line(quoted, start_line) {trailing_comments, comments} = - Enum.split_with(comments, & &1.line > start_line and &1.line < end_line) + Enum.split_with(comments, &(&1.line > start_line and &1.line < end_line)) value = append_comments(value, :trailing_comments, trailing_comments) @@ -250,17 +329,20 @@ defmodule Code.Comments do {quoted, comments} end - {quoted, %{state | parent_doend_meta: [], comments: comments}} + {quoted, %{state | parent_meta: [], comments: comments}} end - # 2-tuples - defp merge_trailing_comments({:__block__, _, [{left, right}]} = quoted, %{comments: comments} = state) when is_tuple(left) and is_tuple(right) do + defp merge_mixed_comments( + {:__block__, _, [{left, right}]} = quoted, + %{comments: comments} = state + ) + when is_tuple(left) and is_tuple(right) do start_line = get_line(right) end_line = get_end_line(quoted, start_line) {trailing_comments, comments} = - Enum.split_with(comments, & &1.line > start_line and &1.line < end_line) + Enum.split_with(comments, &(&1.line > start_line and &1.line < end_line)) right = append_comments(right, :trailing_comments, trailing_comments) @@ -269,23 +351,43 @@ defmodule Code.Comments do end # Stabs - defp merge_trailing_comments({:->, _, [left, right]} = quoted, state) do + defp merge_mixed_comments({:->, _, [left, right]} = quoted, state) do start_line = get_line(right) - end_line = get_end_line({:__block__, state.parent_doend_meta, [quoted]}, start_line) + end_line = get_end_line({:__block__, state.parent_meta, [quoted]}, start_line) + block_start = get_line({:__block__, state.parent_meta, [quoted]}) {right, comments} = case right do {:__block__, _, _} -> - merge_block_trailing_comments(right, start_line, end_line, state.comments) + merge_block_comments(right, start_line, end_line, state.comments) - call -> - line = get_line(call) - {trailing_comments, comments} = - Enum.split_with(state.comments, & &1.line > line and &1.line < end_line) + {_, meta, _} = call -> + if !meta[:trailing_comments] do + line = get_line(call) - call = append_comments(call, :trailing_comments, trailing_comments) + {trailing_comments, comments} = + Enum.split_with(state.comments, &(&1.line > line and &1.line < end_line)) - {call, comments} + call = append_comments(call, :trailing_comments, trailing_comments) + + {call, comments} + else + {right, state.comments} + end + end + + {quoted, comments} = + case left do + [] -> + {leading_comments, comments} = + Enum.split_with(comments, &(&1.line > block_start and &1.line < start_line)) + + quoted = append_comments(quoted, :leading_comments, leading_comments) + + {quoted, comments} + + _ -> + {quoted, comments} end quoted = put_args(quoted, [left, right]) @@ -294,24 +396,49 @@ defmodule Code.Comments do end # Calls - defp merge_trailing_comments({_, meta, args} = quoted, %{comments: comments} = state) when is_list(args) and meta != [] do + defp merge_mixed_comments({form, meta, args} = quoted, state) + when is_list(args) and meta != [] do + with true <- is_atom(form), + <<"sigil_", _name::binary>> <- Atom.to_string(form), + true <- not is_nil(meta) do + [content, modifiers] = args + + {content, state} = merge_mixed_comments(content, state) + + quoted = put_args(quoted, [content, modifiers]) + + {quoted, state} + else + _ -> + merge_call_comments(quoted, state) + end + end + + defp merge_mixed_comments(quoted, state) do + {quoted, state} + end + + defp merge_call_comments({_, meta, quoted_args} = quoted, %{comments: comments} = state) do start_line = get_line(quoted) end_line = get_end_line(quoted, start_line) - {last_arg, args} = List.pop_at(args, -1) + {last_arg, args} = List.pop_at(quoted_args, -1) meta_keys = Keyword.keys(meta) state = - if Enum.any?([:do, :closing], &(&1 in meta_keys)) do - %{state | parent_doend_meta: meta} + if Enum.any?([:do, :end, :closing], &(&1 in meta_keys)) do + %{state | parent_meta: meta} else state end {quoted, comments} = - case last_arg do + case last_arg do [{{:__block__, _, [name]}, _block_args} | _] = blocks when name in @block_names -> - {reversed_blocks, comments} = each_merge_named_block_trailing_comments(blocks, quoted, comments, []) + # For do/end and else/catch/rescue/after blocks, we need to merge the comments + # of each block with the arguments block. + {reversed_blocks, comments} = + each_merge_named_block_comments(blocks, quoted, comments, []) last_arg = Enum.reverse(reversed_blocks) @@ -320,13 +447,21 @@ defmodule Code.Comments do {quoted, comments} + {:->, _, _} -> + {args, comments} = + merge_stab_clause_comments(quoted_args, start_line, end_line, comments, []) + + quoted = put_args(quoted, args) + + {quoted, comments} + [{_key, _value} | _] = pairs -> # Partial keyword list {{last_key, last_value}, pairs} = List.pop_at(pairs, -1) line = get_line(last_value) {trailing_comments, comments} = - Enum.split_with(comments, & &1.line > line and &1.line < end_line) + Enum.split_with(comments, &(&1.line > line and &1.line < end_line)) last_value = append_comments(last_value, :trailing_comments, trailing_comments) @@ -338,20 +473,57 @@ defmodule Code.Comments do {quoted, comments} - {form, _, _} when form != :-> -> - line = get_line(last_arg) + {:__block__, _, [{_, _, _} | _] = block_args} = block when args == [] -> + # This handles cases where the last argument for a call is a block, for example: + # + # assert ( + # # comment + # hello + # world + # ) + # + + {last_block_arg, block_args} = List.pop_at(block_args, -1) + + start_line = get_line(last_block_arg) + {trailing_comments, comments} = - Enum.split_with(comments, & &1.line > line and &1.line < end_line) + Enum.split_with(comments, &(&1.line > start_line and &1.line < end_line)) - last_arg = append_comments(last_arg, :trailing_comments, trailing_comments) + last_block_arg = append_comments(last_block_arg, :trailing_comments, trailing_comments) - args = args ++ [last_arg] - quoted = put_args(quoted, args) + block_args = block_args ++ [last_block_arg] + + block = put_args(block, block_args) + + quoted = put_args(quoted, [block]) {quoted, comments} + + {:__block__, _, [args]} when is_list(args) -> + {quoted, comments} + + {form, _, _} -> + if match?({_, _}, Code.Identifier.binary_op(form)) do + {quoted, comments} + else + line = get_end_line(last_arg, get_line(last_arg)) + + {trailing_comments, comments} = + Enum.split_with(comments, &(&1.line > line and &1.line < end_line)) + + last_arg = append_comments(last_arg, :trailing_comments, trailing_comments) + + args = args ++ [last_arg] + + quoted = put_args(quoted, args) + + {quoted, comments} + end + nil -> {trailing_comments, comments} = - Enum.split_with(comments, & &1.line > start_line and &1.line < end_line) + Enum.split_with(comments, &(&1.line > start_line and &1.line < end_line)) quoted = append_comments(quoted, :inner_comments, trailing_comments) {quoted, comments} @@ -363,49 +535,148 @@ defmodule Code.Comments do {quoted, %{state | comments: comments}} end - defp merge_trailing_comments(quoted, state) do + defp merge_interpolation_comments( + {:"::", interpolation_meta, [{dot_call, inner_meta, [value]}, modifier]} = interpolation, + state + ) do + start_line = get_line(interpolation) + end_line = get_end_line(interpolation, start_line) + value_line = get_line(value) + + {leading_comments, comments} = + Enum.split_with(state.comments, &(&1.line > start_line and &1.line <= value_line)) + + {trailing_comments, comments} = + Enum.split_with(comments, &(&1.line > value_line and &1.line < end_line)) + + value = put_leading_comments(value, leading_comments) + value = put_trailing_comments(value, trailing_comments) + + interpolation = {:"::", interpolation_meta, [{dot_call, inner_meta, [value]}, modifier]} + + {interpolation, %{state | comments: comments}} + end + + defp merge_interpolation_comments( + {{:., dot_meta, [Kernel, :to_string]}, interpolation_meta, [value]} = interpolation, + state + ) do + start_line = get_line(interpolation) + end_line = get_end_line(interpolation, start_line) + value_line = get_line(value) + + {leading_comments, comments} = + Enum.split_with(state.comments, &(&1.line > start_line and &1.line <= value_line)) + + {trailing_comments, comments} = + Enum.split_with(comments, &(&1.line > value_line and &1.line < end_line)) + + value = put_leading_comments(value, leading_comments) + value = put_trailing_comments(value, trailing_comments) + + interpolation = {{:., dot_meta, [Kernel, :to_string]}, interpolation_meta, [value]} + + {interpolation, %{state | comments: comments}} + end + + defp merge_interpolation_comments(quoted, state) do {quoted, state} end - defp each_merge_named_block_trailing_comments([], _, comments, acc), do: {acc, comments} + defp merge_map_args_trailing_comments(quoted, args, %{comments: comments} = state) do + case List.pop_at(args, -1) do + {{last_key, last_value}, args} -> + start_line = get_line(last_value) + end_line = get_end_line(quoted, start_line) + + {trailing_comments, comments} = + Enum.split_with(comments, &(&1.line > start_line and &1.line < end_line)) + + last_value = append_comments(last_value, :trailing_comments, trailing_comments) + + args = args ++ [{last_key, last_value}] + + {args, %{state | comments: comments}} + + {{:unquote_splicing, _, _} = unquote_splicing, other} -> + start_line = get_line(unquote_splicing) + end_line = get_end_line(quoted, start_line) + + {trailing_comments, comments} = + Enum.split_with(comments, &(&1.line > start_line and &1.line < end_line)) + + unquote_splicing = + append_comments(unquote_splicing, :trailing_comments, trailing_comments) + + args = other ++ [unquote_splicing] + + {args, %{state | comments: comments}} + end + end + + defp each_merge_named_block_comments([], _, comments, acc), do: {acc, comments} - defp each_merge_named_block_trailing_comments([{block, block_args} | rest], parent, comments, acc) do + defp each_merge_named_block_comments([{block, block_args} | rest], parent, comments, acc) do block_start = get_line(block) + block_end = case rest do [{next_block, _} | _] -> + # The parent node only has metadata about the `do` and `end` token positions, + # but in order to know when each individual block ends, we need to look at the + # next block. get_line(next_block) + [] -> + # If there is no next block, we can assume the `end` token follows, so we use + # the parent node's end position. get_end_line(parent, 0) end - {block, block_args, comments} = merge_named_block_trailing_comments(block, block_args, block_start, block_end, comments) + {block, block_args, comments} = + merge_named_block_comments(block, block_args, block_start, block_end, comments) acc = [{block, block_args} | acc] - each_merge_named_block_trailing_comments(rest, parent, comments, acc) + each_merge_named_block_comments(rest, parent, comments, acc) end - defp merge_named_block_trailing_comments(block, {_, _, args} = block_args, block_start, block_end, comments) when is_list(args) do + defp merge_named_block_comments( + block, + {_, _, args} = block_args, + block_start, + block_end, + comments + ) + when is_list(args) do {last_arg, args} = List.pop_at(args, -1) case last_arg do nil -> {trailing_comments, comments} = - Enum.split_with(comments, & &1.line > block_start and &1.line < block_end) + Enum.split_with(comments, &(&1.line > block_start and &1.line < block_end)) block_args = append_comments(block_args, :inner_comments, trailing_comments) - {block, block_args, comments} + {block, block_args, comments} last_arg when not is_list(last_arg) -> - {last_arg, comments} = - merge_trailing_comments_to_last_arg(last_arg, block_start, block_end, comments) + case last_arg do + {:__block__, _, [args]} when is_list(args) -> + # It it's a list, we skip merging comments to avoid collecting all trailing + # comments into the list metadata. Otherwise, we will not be able to collect + # leading comments for the individual elements in the list. + {block, block_args, comments} - args = args ++ [last_arg] - block_args = put_args(block_args, args) + _ -> + {last_arg, comments} = + merge_comments_to_last_arg(last_arg, block_start, block_end, comments) - {block, block_args, comments} + args = args ++ [last_arg] + block_args = put_args(block_args, args) + + {block, block_args, comments} + end _ -> {block, block_args, comments} @@ -414,33 +685,107 @@ defmodule Code.Comments do # If a do/end block has a single argument, it will not be wrapped in a `:__block__` node, # so we need to check for that. - defp merge_named_block_trailing_comments(block, {_, _, ctx} = single_arg, block_start, block_end, comments) when not is_list(ctx) do + defp merge_named_block_comments( + block, + {_, _, ctx} = single_arg, + block_start, + block_end, + comments + ) + when not is_list(ctx) do {last_arg, comments} = - merge_trailing_comments_to_last_arg(single_arg, block_start, block_end, comments) + merge_comments_to_last_arg(single_arg, block_start, block_end, comments) {block, last_arg, comments} end - defp merge_named_block_trailing_comments(block, block_args, _, _, comments), - do: {block, block_args, comments} + defp merge_named_block_comments( + block, + [{:->, _, _} | _] = block_args, + block_start, + block_end, + comments + ) do + {block_args, comments} = + merge_stab_clause_comments(block_args, block_start, block_end, comments, []) + + {block, block_args, comments} + end + + defp merge_stab_clause_comments( + [{:->, _stab_meta, [left, right]} = stab | rest], + block_start, + block_end, + comments, + acc + ) do + start_line = get_line(right) + + end_line = + case rest do + [{:->, _, _} | _] -> + get_end_line(right, start_line) + + [] -> + block_end + end + + {stab, comments} = + case left do + [] -> + stab_line = get_line(stab) + + {leading_comments, comments} = + Enum.split_with(comments, &(&1.line > block_start and &1.line < stab_line)) + + stab = append_comments(stab, :leading_comments, leading_comments) + + {stab, comments} + + _ -> + {stab, comments} + end + + {right, comments} = + case right do + {:__block__, _, _} -> + merge_block_comments(right, start_line, end_line, comments) + + call -> + {trailing_comments, comments} = + Enum.split_with(comments, &(&1.line > start_line and &1.line < end_line)) + + call = append_comments(call, :trailing_comments, trailing_comments) + + {call, comments} + end + + stab = put_args(stab, [left, right]) + + acc = [stab | acc] + + merge_stab_clause_comments(rest, block_start, block_end, comments, acc) + end + + defp merge_stab_clause_comments([], _, _, comments, acc), do: {Enum.reverse(acc), comments} - defp merge_block_trailing_comments({:__block__, _, args} = block, block_start, block_end, comments) do + defp merge_block_comments({:__block__, _, args} = block, block_start, block_end, comments) do {last_arg, args} = List.pop_at(args, -1) case last_arg do nil -> {trailing_comments, comments} = - Enum.split_with(comments, & &1.line > block_start and &1.line < block_end) + Enum.split_with(comments, &(&1.line > block_start and &1.line < block_end)) trailing_comments = Enum.sort_by(trailing_comments, & &1.line) block = append_comments(block, :inner_comments, trailing_comments) - {block, comments} + {block, comments} last_arg when not is_list(last_arg) -> {last_arg, comments} = - merge_trailing_comments_to_last_arg(last_arg, block_start, block_end, comments) + merge_comments_to_last_arg(last_arg, block_start, block_end, comments) args = args ++ [last_arg] block = put_args(block, args) @@ -449,7 +794,7 @@ defmodule Code.Comments do inner_list when is_list(inner_list) -> {inner_list, comments} = - merge_trailing_comments_to_last_arg(inner_list, block_start, block_end, comments) + merge_comments_to_last_arg(inner_list, block_start, block_end, comments) args = args ++ [inner_list] block = put_args(block, args) @@ -461,17 +806,17 @@ defmodule Code.Comments do end end - defp merge_trailing_comments_to_last_arg(last_arg, block_start, block_end, comments) do - line = - case last_arg do - [] -> block_start - [first | _] -> get_line(first) - {_, _, _} -> get_line(last_arg) - _ -> block_start - end + defp merge_comments_to_last_arg(last_arg, block_start, block_end, comments) do + line = + case last_arg do + [] -> block_start + [first | _] -> get_line(first) + {_, _, _} -> get_line(last_arg) + _ -> block_start + end {trailing_comments, comments} = - Enum.split_with(comments, & &1.line > line and &1.line < block_end) + Enum.split_with(comments, &(&1.line > line and &1.line < block_end)) last_arg = append_comments(last_arg, :trailing_comments, trailing_comments) @@ -492,7 +837,6 @@ defmodule Code.Comments do defp get_meta({_, meta, _}) when is_list(meta), do: meta - defp get_line({_, meta, _}, default \\ 1) when is_list(meta) and (is_integer(default) or is_nil(default)) do Keyword.get(meta, :line, default) @@ -586,4 +930,12 @@ defmodule Code.Comments do defp put_args({form, meta, _args}, args) do {form, meta, args} end + + defp interpolated?(entries) do + Enum.all?(entries, fn + {:"::", _, [{{:., _, [Kernel, :to_string]}, _, [_]}, {:binary, _, _}]} -> true + entry when is_binary(entry) -> true + _ -> false + end) + end end diff --git a/lib/elixir/lib/code/formatter.ex b/lib/elixir/lib/code/formatter.ex index 80124657bf..6131e320d8 100644 --- a/lib/elixir/lib/code/formatter.ex +++ b/lib/elixir/lib/code/formatter.ex @@ -159,13 +159,16 @@ defmodule Code.Formatter do Converts the quoted expression into an algebra document. """ def to_algebra(quoted, opts \\ []) do - comments = Keyword.get(opts, :comments, []) + state = state(opts) - state = - comments - |> Enum.map(&format_comment/1) - |> gather_comments() - |> state(opts) + comments = opts[:comments] + + quoted = + if is_list(comments) do + Code.Comments.merge_comments(quoted, comments) + else + quoted + end {doc, _} = block_to_algebra(quoted, @min_line, @max_line, state) doc @@ -188,7 +191,7 @@ defmodule Code.Formatter do end) end - defp state(comments, opts) do + defp state(opts) do force_do_end_blocks = Keyword.get(opts, :force_do_end_blocks, false) locals_without_parens = Keyword.get(opts, :locals_without_parens, []) file = Keyword.get(opts, :file, nil) @@ -219,7 +222,6 @@ defmodule Code.Formatter do locals_without_parens: locals_without_parens ++ locals_without_parens(), operand_nesting: 2, skip_eol: false, - comments: comments, sigils: sigils, file: file, migrate_bitstring_modifiers: migrate_bitstring_modifiers, @@ -240,36 +242,6 @@ defmodule Code.Formatter do defp format_comment_text("# " <> rest), do: "# " <> rest defp format_comment_text("#" <> rest), do: "# " <> rest - # If there is a no new line before, we can't gather all followup comments. - defp gather_comments([%{previous_eol_count: 0} = comment | comments]) do - comment = %{comment | previous_eol_count: @newlines} - [comment | gather_comments(comments)] - end - - defp gather_comments([comment | comments]) do - %{line: line, next_eol_count: next_eol_count, text: doc} = comment - - {next_eol_count, comments, doc} = - gather_followup_comments(line + 1, next_eol_count, comments, doc) - - comment = %{comment | next_eol_count: next_eol_count, text: doc} - [comment | gather_comments(comments)] - end - - defp gather_comments([]) do - [] - end - - defp gather_followup_comments(line, _, [%{line: line} = comment | comments], doc) - when comment.previous_eol_count != 0 do - %{next_eol_count: next_eol_count, text: text} = comment - gather_followup_comments(line + 1, next_eol_count, comments, line(doc, text)) - end - - defp gather_followup_comments(_line, next_eol_count, comments, doc) do - {next_eol_count, comments, doc} - end - # Special AST nodes from compiler feedback defp quoted_to_algebra({{:special, :clause_args}, _meta, [args]}, _context, state) do @@ -641,23 +613,46 @@ defmodule Code.Formatter do paren_fun_to_algebra(paren_fun, min_line, max_line, state) end - defp block_to_algebra({:__block__, meta, args}, _min_line, _max_line, state) when args in [[], [nil]] do + defp block_to_algebra({:__block__, meta, []}, _min_line, _max_line, state) do inner_comments = meta[:inner_comments] || [] + comments_docs = Enum.map(inner_comments, fn comment -> comment = format_comment(comment) {comment.text, @empty, 1} end) - docs = merge_algebra_with_comments(comments_docs, @empty) + comments_docs = merge_algebra_with_comments(comments_docs, @empty) - case docs do + case comments_docs do [] -> {@empty, state} [line] -> {line, state} lines -> {lines |> Enum.reduce(&line(&2, &1)) |> force_unfit(), state} end end + defp block_to_algebra({:__block__, meta, [nil]} = block, min_line, max_line, state) do + inner_comments = meta[:inner_comments] || [] + + comments_docs = + Enum.map(inner_comments, fn comment -> + comment = format_comment(comment) + {comment.text, @empty, 1} + end) + + comments_docs = merge_algebra_with_comments(comments_docs, @empty) + + {doc, state} = block_args_to_algebra([block], min_line, max_line, state) + + docs = case comments_docs do + [] -> doc + [line] -> line(doc, line) + lines -> doc |> line(lines |> Enum.reduce(&line(&2, &1)) |> force_unfit()) + end + + {docs, state} + end + defp block_to_algebra({:__block__, _, [_, _ | _] = args}, min_line, max_line, state) do block_args_to_algebra(args, min_line, max_line, state) end @@ -1199,7 +1194,7 @@ defmodule Code.Formatter do # defp call_args_to_algebra([], meta, _context, _parens, _list_to_keyword?, state) do {args_doc, _join, state} = - args_to_algebra_with_comments([], meta, false, :none, :break, state, &{&1, &2}) + args_to_algebra_with_comments([], meta, :none, :break, state, &{&1, &2}) {{surround("(", args_doc, ")"), state}, false} end @@ -1238,7 +1233,7 @@ defmodule Code.Formatter do defp call_args_to_algebra_no_blocks(meta, args, skip_parens?, list_to_keyword?, extra, state) do {left, right} = split_last(args) - {keyword?, right} = last_arg_to_keyword(right, list_to_keyword?, skip_parens?, state.comments) + {keyword?, right} = last_arg_to_keyword(right, list_to_keyword?, skip_parens?) context = if left == [] and not keyword? do @@ -1260,7 +1255,6 @@ defmodule Code.Formatter do args_to_algebra_with_comments( left, Keyword.delete(meta, :closing), - skip_parens?, :force_comma, join, state, @@ -1270,7 +1264,7 @@ defmodule Code.Formatter do join = if force_args?(right) or force_args?(args) or many_eol?, do: :line, else: :break {right_doc, _join, state} = - args_to_algebra_with_comments(right, meta, false, :none, join, state, to_algebra_fun) + args_to_algebra_with_comments(right, meta, :none, join, state, to_algebra_fun) right_doc = apply(Inspect.Algebra, join, []) |> concat(right_doc) @@ -1300,7 +1294,6 @@ defmodule Code.Formatter do args_to_algebra_with_comments( args, meta, - skip_parens?, last_arg_mode, join, state, @@ -1523,7 +1516,7 @@ defmodule Code.Formatter do {args_doc, join, state} = args |> Enum.with_index() - |> args_to_algebra_with_comments(meta, false, :none, join, state, to_algebra_fun) + |> args_to_algebra_with_comments(meta, :none, join, state, to_algebra_fun) if join == :flex_break do {"<<" |> concat(args_doc) |> nest(2) |> concat(">>") |> group(), state} @@ -1618,7 +1611,7 @@ defmodule Code.Formatter do fun = "ed_to_algebra(&1, :parens_arg, &2) {args_doc, _join, state} = - args_to_algebra_with_comments(args, meta, false, :none, join, state, fun) + args_to_algebra_with_comments(args, meta, :none, join, state, fun) left_bracket = color_doc("[", :list, state.inspect_opts) right_bracket = color_doc("]", :list, state.inspect_opts) @@ -1632,7 +1625,7 @@ defmodule Code.Formatter do {left_doc, state} = fun.(left, state) {right_doc, _join, state} = - args_to_algebra_with_comments(right, meta, false, :none, join, state, fun) + args_to_algebra_with_comments(right, meta, :none, join, state, fun) args_doc = left_doc @@ -1647,7 +1640,7 @@ defmodule Code.Formatter do fun = "ed_to_algebra(&1, :parens_arg, &2) {args_doc, _join, state} = - args_to_algebra_with_comments(args, meta, false, :none, join, state, fun) + args_to_algebra_with_comments(args, meta, :none, join, state, fun) do_map_to_algebra(name_doc, args_doc, state) end @@ -1662,7 +1655,7 @@ defmodule Code.Formatter do fun = "ed_to_algebra(&1, :parens_arg, &2) {args_doc, join, state} = - args_to_algebra_with_comments(args, meta, false, :none, join, state, fun) + args_to_algebra_with_comments(args, meta, :none, join, state, fun) left_bracket = color_doc("{", :tuple, state.inspect_opts) right_bracket = color_doc("}", :tuple, state.inspect_opts) @@ -1802,7 +1795,7 @@ defmodule Code.Formatter do defp heredoc_line(["\r", _ | _]), do: nest(line(), :reset) defp heredoc_line(_), do: line() - defp args_to_algebra_with_comments(args, meta, skip_parens?, last_arg_mode, join, state, fun) do + defp args_to_algebra_with_comments(args, meta, last_arg_mode, join, state, fun) do min_line = line(meta) max_line = closing_line(meta) @@ -1827,25 +1820,26 @@ defmodule Code.Formatter do {{doc, @empty, 1}, state} end - # If skipping parens, we cannot extract the comments of the first - # argument as there is no place to move them to, so we handle it now. + inner_comments = List.wrap(meta[:inner_comments]) + comments_doc = + Enum.map(inner_comments, fn comment -> + comment = format_comment(comment) + {comment.text, @empty, 1} + end) + + join = if args == [] and inner_comments != [], do: :line, else: join + {args, acc, state} = case args do - [head | tail] when skip_parens? -> - {doc_triplet, state} = arg_to_algebra.(head, tail, state) - {tail, [doc_triplet], state} + [] -> + {args, comments_doc, state} - _ -> + [_ | _] -> {args, [], state} end {args_docs, comments?, state} = - case args do - [] -> - quoted_to_algebra_with_comments(args, acc, min_line, max_line, state, arg_to_algebra) - _ -> - quoted_to_algebra_with_comments(args, acc, min_line, max_line, state, arg_to_algebra) - end + quoted_to_algebra_with_comments(args, acc, min_line, max_line, state, arg_to_algebra) cond do args_docs == [] -> @@ -1886,6 +1880,22 @@ defmodule Code.Formatter do |> maybe_force_clauses(clauses, state) |> group() + leading_comments = meta[:leading_comments] || [] + + comments = + Enum.map(leading_comments, fn comment -> + comment = format_comment(comment) + {comment.text, @empty, 1} + end) + + comments = merge_algebra_with_comments(comments, @empty) + + # If there are any comments before the ->, we hoist them up above the fn + doc = case comments do + [] -> doc + [comments] -> line(comments, doc) + end + {doc, state} end @@ -2119,48 +2129,214 @@ defmodule Code.Formatter do {acc, comments?, state} end - defp each_quoted_to_algebra_with_comments([arg | args], acc, state, fun, comments?) do + defp each_quoted_to_algebra_with_comments([arg | args], acc, state, fun, _comments?) do + {doc_start, doc_end} = traverse_line(arg, {@max_line, @min_line}) + {leading_comments, trailing_comments, arg} = extract_arg_comments(arg) + + comments? = leading_comments != [] or trailing_comments != [] + + leading_docs = build_leading_comments([], leading_comments, doc_start) + trailing_docs = build_trailing_comments([], trailing_comments) + + next_comments = + case args do + [next_arg | _] -> + {next_leading_comments, _, _} = extract_arg_comments(next_arg) + next_leading_comments ++ trailing_comments + + [] -> + trailing_comments + end + {doc_triplet, state} = fun.(arg, args, state) + doc_triplet = adjust_trailing_newlines(doc_triplet, doc_end, next_comments) - case traverse_line(arg, {@max_line, @min_line}) do - {@max_line, @min_line} -> - acc = [doc_triplet | acc] - each_quoted_to_algebra_with_comments(args, acc, state, fun, comments?) + acc = Enum.concat([trailing_docs, [doc_triplet], leading_docs, acc]) - {doc_start, doc_end} -> - {leading_comments, trailing_comments} = - case arg do - {_, meta, _} -> - leading_comments = meta[:leading_comments] || [] - trailing_comments = meta[:trailing_comments] || [] - {leading_comments, trailing_comments} + each_quoted_to_algebra_with_comments(args, acc, state, fun, comments?) + end - {{_, left_meta, _}, {_, right_meta, _}} -> - leading_comments = left_meta[:leading_comments] || [] - trailing_comments = right_meta[:trailing_comments] || [] + defp extract_arg_comments([{_, _, _} | _] = arg) do + {leading_comments, trailing_comments} = + Enum.reduce(arg, {[], []}, fn {_, _, _} = item, {leading_comments, trailing_comments} -> + {item_leading_comments, item_trailing_comments} = extract_comments(item) - {leading_comments, trailing_comments} - _ -> - {[], []} - end + {leading_comments ++ item_leading_comments, trailing_comments ++ item_trailing_comments} + end) + + {leading_comments, trailing_comments, arg} + end + + defp extract_arg_comments({{_, _, _} = quoted, index} = arg) when is_integer(index) do + {leading_comments, trailing_comments} = extract_comments(quoted) + {leading_comments, trailing_comments, arg} + end + + defp extract_arg_comments({:<<>>, _, _} = arg) do + extract_interpolation_comments(arg) + end - comments? = leading_comments != [] or trailing_comments != [] + defp extract_arg_comments({{:., _, [List, :to_charlist]}, _, _} = arg) do + extract_interpolation_comments(arg) + end - leading_docs = build_leading_comments([], leading_comments, doc_start) - trailing_docs = build_trailing_comments([], trailing_comments) + defp extract_arg_comments({_, _, _} = arg) do + {leading_comments, trailing_comments} = extract_comments(arg) + {leading_comments, trailing_comments, arg} + end - doc_triplet = adjust_trailing_newlines(doc_triplet, doc_end, trailing_comments) + defp extract_arg_comments({{_, _, _} = left, {_, _, _} = right} = arg) do + {leading_comments, _} = extract_comments(left) + {_, trailing_comments} = extract_comments(right) + + {leading_comments, trailing_comments, arg} + end + + defp extract_arg_comments({{_, context}, {_, _, _} = quoted} = arg) when context in [:left, :right, :operand, :parens_arg] do + {leading_comments, trailing_comments} = extract_comments(quoted) + {leading_comments, trailing_comments, arg} + end + + defp extract_arg_comments(arg) do + {[], [], arg} + end - acc = Enum.concat([trailing_docs, [doc_triplet], leading_docs, acc]) + defp extract_comments({_, meta, _}) do + leading = List.wrap(meta[:leading_comments]) + trailing = List.wrap(meta[:trailing_comments]) + + {leading, trailing} + end + + defp extract_comments(_), do: {[], []} + + defp extract_comments_within(quoted) do + {_, comments} = + Macro.postwalk(quoted, [], fn + {_, _, _} = quoted, acc -> + {leading, trailing} = extract_comments(quoted) + acc = Enum.concat([acc, leading, trailing]) + {quoted, acc} + + other, acc -> + {other, acc} + end) + + Enum.sort_by(comments, & &1.line) + end - each_quoted_to_algebra_with_comments(args, acc, state, fun, comments?) + defp extract_interpolation_comments({:<<>>, meta, entries} = quoted) when is_list(entries) do + {node_leading, node_trailing} = extract_comments(quoted) + + if interpolated?(entries) do + {entries, comments} = + Macro.postwalk(entries, [], fn + {form, meta, args} = entry, acc -> + {leading, trailing} = extract_comments(entry) + + acc = Enum.concat([leading, trailing, acc]) + meta = Keyword.drop(meta, [:leading_comments, :trailing_comments]) + quoted = {form, meta, args} + + {quoted, acc} + + quoted, acc -> + {quoted, acc} + end) + + quoted = {:<<>>, meta, entries} + + comments = Enum.sort_by(comments, & &1.line) + + last_value = + for {:"::", _, [{_, _, [last_value]}, _]} <- entries, reduce: nil do + _ -> last_value + end + + {_, last_node_line} = + Macro.postwalk(last_value, 0, fn + {_, meta, _} = quoted, max_seen -> + line = meta[:line] || max_seen + + {quoted, max(max_seen, line)} + + quoted, max_seen -> + {quoted, max_seen} + end) + + {leading, trailing} = + Enum.split_with(comments, fn comment -> + comment.line <= last_node_line + end) + + {node_leading ++ leading, node_trailing ++ trailing, quoted} + + else + {node_leading, node_trailing, quoted} + end + end + + defp extract_interpolation_comments({{:., _, [List, :to_charlist]} = dot, meta, [entries]} = quoted) when is_list(entries) do + {node_leading, node_trailing} = extract_comments(quoted) + + if list_interpolated?(entries) && meta[:delimiter] do + {entries, comments} = + Macro.postwalk(entries, [], fn + {form, meta, args} = entry, acc -> + {leading, trailing} = extract_comments(entry) + + acc = Enum.concat([leading, trailing, acc]) + meta = Keyword.drop(meta, [:leading_comments, :trailing_comments]) + quoted = {form, meta, args} + + {quoted, acc} + + quoted, acc -> + {quoted, acc} + end) + + quoted = {dot, meta, [entries]} + + comments = Enum.sort_by(comments, & &1.line) + + last_value = + for {{:., _, [Kernel, :to_string]}, _, [last_value]} <- entries, reduce: nil do + _ -> last_value + end + + {_, last_node_line} = + Macro.postwalk(last_value, 0, fn + {_, meta, _} = quoted, max_seen -> + line = meta[:line] || max_seen + + {quoted, max(max_seen, line)} + + quoted, max_seen -> + {quoted, max_seen} + end) + + {leading, trailing} = + Enum.split_with(comments, fn comment -> + comment.line <= last_node_line + end) + + {node_leading ++ leading, node_trailing ++ trailing, quoted} + + else + {node_leading, node_trailing, quoted} end end - defp build_leading_comments(acc, [], _), do: acc + defp extract_interpolation_comments(quoted), do: {[], [], quoted} - defp build_leading_comments(acc, [comment | rest], doc_start) do + defp build_leading_comments(acc, comments, doc_start) do + do_build_leading_comments(acc, comments, doc_start) + end + + defp do_build_leading_comments(acc, [], _), do: acc + + defp do_build_leading_comments(acc, [comment | rest], doc_start) do comment = format_comment(comment) %{previous_eol_count: previous, next_eol_count: next, text: doc, line: line} = comment # If the comment is on the same line as the document, we need to adjust the newlines @@ -2170,8 +2346,9 @@ defmodule Code.Formatter do build_leading_comments(acc, rest, doc_start) end - defp add_previous_to_acc([{doc, next_line, newlines} | acc], previous) when newlines < previous, - do: [{doc, next_line, previous} | acc] + defp add_previous_to_acc([{doc, next_line, newlines} | acc], previous) when newlines < previous do + [{doc, next_line, previous} | acc] + end defp add_previous_to_acc(acc, _previous), do: acc @@ -2179,7 +2356,7 @@ defmodule Code.Formatter do defp build_trailing_comments(acc, [comment | rest]) do comment = format_comment(comment) - %{previous_eol_count: previous, next_eol_count: next, text: doc} = comment + %{next_eol_count: next, text: doc} = comment acc = [{doc, @empty, next} | acc] build_trailing_comments(acc, rest) end @@ -2191,6 +2368,13 @@ defmodule Code.Formatter do {doc, next_line, 1} end + # If the document is followed by newlines and then a comment, we need to adjust the + # newlines such that there is an empty line between the document and the comments. + defp adjust_trailing_newlines({doc, next_line, newlines}, doc_end, [%{line: line} | _]) + when newlines <= 1 and line > doc_end + 1 do + {doc, next_line, 2} + end + defp adjust_trailing_newlines(doc_triplet, _, _), do: doc_triplet @@ -2399,17 +2583,18 @@ defmodule Code.Formatter do false end - defp eol_or_comments?(meta, %{comments: comments} = state) do + defp eol_or_comments?(meta, state) do eol?(meta, state) or ( min_line = line(meta) max_line = closing_line(meta) + comments = meta[:trailing_comments] || [] Enum.any?(comments, fn %{line: line} -> line > min_line and line < max_line end) ) end # A literal list is a keyword or (... -> ...) - defp last_arg_to_keyword([_ | _] = arg, _list_to_keyword?, _skip_parens?, _comments) do + defp last_arg_to_keyword([_ | _] = arg, _list_to_keyword?, _skip_parens?) do {keyword?(arg), arg} end @@ -2417,8 +2602,7 @@ defmodule Code.Formatter do defp last_arg_to_keyword( {:__block__, meta, [[_ | _] = arg]} = block, true, - skip_parens?, - comments + skip_parens? ) do cond do not keyword?(arg) -> @@ -2429,6 +2613,8 @@ defmodule Code.Formatter do {{_, arg_meta, _}, _} = hd(arg) first_line = line(arg_meta) + comments = extract_comments_within(block) + case Enum.drop_while(comments, fn %{line: line} -> line <= block_line end) do [%{line: line} | _] when line <= first_line -> {false, block} @@ -2443,7 +2629,7 @@ defmodule Code.Formatter do end # Otherwise we don't have a keyword. - defp last_arg_to_keyword(arg, _list_to_keyword?, _skip_parens?, _comments) do + defp last_arg_to_keyword(arg, _list_to_keyword?, _skip_parens?) do {false, arg} end diff --git a/lib/elixir/test/elixir/code/ast_comments_test.exs b/lib/elixir/test/elixir/code/ast_comments_test.exs index bd4a2d2f74..c1a833942f 100644 --- a/lib/elixir/test/elixir/code/ast_comments_test.exs +++ b/lib/elixir/test/elixir/code/ast_comments_test.exs @@ -4,7 +4,7 @@ defmodule Code.AstCommentsTest do use ExUnit.Case, async: true def parse_string!(string) do - Code.string_to_quoted!(string, include_comments: true, emit_warnings: false) + Code.string_to_quoted!(string, include_comments: true, literal_encoder: &{:ok, {:__block__, &2, [&1]}}, emit_warnings: false) end describe "merge_comments/2" do @@ -210,6 +210,7 @@ defmodule Code.AstCommentsTest do #trailing 3 ] # trailing outside """) + |> IO.inspect() assert {:__block__, list_meta, [ @@ -647,5 +648,28 @@ defmodule Code.AstCommentsTest do assert [%{line: 10, text: "# after body"}] = world_meta[:trailing_comments] end + + test "merges leading comments into the stab if left side is empty" do + quoted = + parse_string!(""" + fn + # leading + -> + hello + hello + end + """) + + assert {:fn, _, + [ + {:->, stab_meta, + [ + [], + _ + ]} + ]} = quoted + + assert [%{line: 2, text: "# leading"}] = stab_meta[:leading_comments] + end end end diff --git a/lib/elixir/test/elixir/code_formatter/ast_comments_test.exs b/lib/elixir/test/elixir/code_formatter/ast_comments_test.exs deleted file mode 100644 index 576f6fab83..0000000000 --- a/lib/elixir/test/elixir/code_formatter/ast_comments_test.exs +++ /dev/null @@ -1,521 +0,0 @@ -Code.require_file("../test_helper.exs", __DIR__) - -defmodule Code.Formatter.AstCommentsTest do - use ExUnit.Case, async: true - - def parse_string!(string) do - Code.string_to_quoted!(string, include_comments: true) - end - - describe "merge_comments/2" do - test "merges comments in empty AST" do - quoted = - parse_string!(""" - # some comment - # another comment - """) - - assert {:__block__, meta, []} = quoted - - assert [%{line: 1, text: "# some comment"}, %{line: 2, text: "# another comment"}] = - meta[:inner_comments] - end - - test "merges leading comments in assorted terms" do - quoted = - parse_string!(""" - # leading var - var - # trailing var - """) - - assert {:var, meta, _} = quoted - - assert [%{line: 1, text: "# leading var"}] = meta[:leading_comments] - assert [%{line: 3, text: "# trailing var"}] = meta[:trailing_comments] - - quoted = - parse_string!(""" - # leading 1 - 1 - # trailing 1 - """) - - assert {:__block__, one_meta, [1]} = quoted - - assert [%{line: 1, text: "# leading 1"}] = one_meta[:leading_comments] - assert [%{line: 3, text: "# trailing 1"}] = one_meta[:trailing_comments] - - quoted = - parse_string!(""" - # leading qualified call - Foo.bar(baz) - # trailing qualified call - """) - - assert {{:., _, [_Foo, _bar]}, meta, _} = quoted - - assert [%{line: 1, text: "# leading qualified call"}] = meta[:leading_comments] - assert [%{line: 3, text: "# trailing qualified call"}] = meta[:trailing_comments] - - quoted = - parse_string!(""" - # leading qualified call - Foo. - # leading bar - bar(baz) - # trailing qualified call - """) - - assert {{:., _, [_Foo, _]}, meta, - [ - {:baz, _, _} - ]} = quoted - - assert [%{line: 1, text: "# leading qualified call"}, %{line: 3, text: "# leading bar"}] = - meta[:leading_comments] - - assert [%{line: 5, text: "# trailing qualified call"}] = meta[:trailing_comments] - end - - # Do/end blocks - - test "merges comments in do/end block" do - quoted = - parse_string!(""" - def a do - foo() - :ok - # A - end # B - """) - - assert {:def, def_meta, - [ - {:a, _, _}, - [ - {{:__block__, _, [:do]}, - {:__block__, _, - [ - {:foo, _, _}, - {:__block__, meta, [:ok]} - ]}} - ] - ]} = - quoted - - assert [%{line: 4, text: "# A"}] = meta[:trailing_comments] - - assert [%{line: 5, text: "# B"}] = def_meta[:trailing_comments] - end - - test "merges comments for named do/end blocks" do - quoted = - parse_string!(""" - def a do - # leading var1 - var1 - # trailing var1 - else - # leading var2 - var2 - # trailing var2 - catch - # leading var3 - var3 - # trailing var3 - rescue - # leading var4 - var4 - # trailing var4 - after - # leading var5 - var5 - # trailing var5 - end - """) - - assert {:def, _, - [ - {:a, _, _}, - [ - {{:__block__, _, [:do]}, {:var1, var1_meta, _}}, - {{:__block__, _, [:else]}, {:var2, var2_meta, _}}, - {{:__block__, _, [:catch]}, {:var3, var3_meta, _}}, - {{:__block__, _, [:rescue]}, {:var4, var4_meta, _}}, - {{:__block__, _, [:after]}, {:var5, var5_meta, _}} - ] - ]} = - quoted - - assert [%{line: 2, text: "# leading var1"}] = var1_meta[:leading_comments] - assert [%{line: 4, text: "# trailing var1"}] = var1_meta[:trailing_comments] - assert [%{line: 6, text: "# leading var2"}] = var2_meta[:leading_comments] - assert [%{line: 8, text: "# trailing var2"}] = var2_meta[:trailing_comments] - assert [%{line: 10, text: "# leading var3"}] = var3_meta[:leading_comments] - assert [%{line: 12, text: "# trailing var3"}] = var3_meta[:trailing_comments] - assert [%{line: 14, text: "# leading var4"}] = var4_meta[:leading_comments] - assert [%{line: 16, text: "# trailing var4"}] = var4_meta[:trailing_comments] - assert [%{line: 18, text: "# leading var5"}] = var5_meta[:leading_comments] - assert [%{line: 20, text: "# trailing var5"}] = var5_meta[:trailing_comments] - end - - test "merges inner comments for empty named do/end blocks" do - quoted = - parse_string!(""" - def a do - # inside do - else - # inside else - catch - # inside catch - rescue - # inside rescue - after - # inside after - end - """) - - assert {:def, _, - [ - {:a, _, _}, - [ - {{:__block__, _, [:do]}, {:__block__, do_meta, _}}, - {{:__block__, _, [:else]}, {:__block__, else_meta, _}}, - {{:__block__, _, [:catch]}, {:__block__, catch_meta, _}}, - {{:__block__, _, [:rescue]}, {:__block__, rescue_meta, _}}, - {{:__block__, _, [:after]}, {:__block__, after_meta, _}} - ] - ]} = - quoted - - assert [%{line: 2, text: "# inside do"}] = do_meta[:inner_comments] - assert [%{line: 4, text: "# inside else"}] = else_meta[:inner_comments] - assert [%{line: 6, text: "# inside catch"}] = catch_meta[:inner_comments] - assert [%{line: 8, text: "# inside rescue"}] = rescue_meta[:inner_comments] - assert [%{line: 10, text: "# inside after"}] = after_meta[:inner_comments] - end - - # Lists - - test "merges comments in list" do - quoted = - parse_string!(""" - [ - #leading 1 - 1, - #leading 2 - 2, - 3 - #trailing 3 - ] # trailing outside - """) - - assert {:__block__, list_meta, - [ - [ - {:__block__, one_meta, [1]}, - {:__block__, two_meta, [2]}, - {:__block__, three_meta, [3]} - ] - ]} = quoted - - assert [%{line: 2, text: "#leading 1"}] = one_meta[:leading_comments] - assert [%{line: 4, text: "#leading 2"}] = two_meta[:leading_comments] - assert [%{line: 7, text: "#trailing 3"}] = three_meta[:trailing_comments] - assert [%{line: 8, text: "# trailing outside"}] = list_meta[:trailing_comments] - end - - test "merges inner comments in empty list" do - quoted = - parse_string!(""" - [ - # inner 1 - # inner 2 - ] # trailing outside - """) - - assert {:__block__, list_meta, [[]]} = quoted - - assert [%{line: 2, text: "# inner 1"}, %{line: 3, text: "# inner 2"}] = - list_meta[:inner_comments] - - assert [%{line: 4, text: "# trailing outside"}] = list_meta[:trailing_comments] - end - - # Keyword lists - - test "merges comments in keyword list" do - quoted = - parse_string!(""" - [ - #leading a - a: 1, - #leading b - b: 2, - c: 3 - #trailing 3 - ] # trailing outside - """) - - assert {:__block__, keyword_list_meta, - [ - [ - { - {:__block__, a_key_meta, [:a]}, - {:__block__, _, [1]} - }, - { - {:__block__, b_key_meta, [:b]}, - {:__block__, _, [2]} - }, - { - {:__block__, _, [:c]}, - {:__block__, c_value_meta, [3]} - } - ] - ]} = quoted - - assert [%{line: 2, text: "#leading a"}] = a_key_meta[:leading_comments] - assert [%{line: 4, text: "#leading b"}] = b_key_meta[:leading_comments] - assert [%{line: 7, text: "#trailing 3"}] = c_value_meta[:trailing_comments] - assert [%{line: 8, text: "# trailing outside"}] = keyword_list_meta[:trailing_comments] - end - - test "merges comments in partial keyword list" do - quoted = - parse_string!(""" - [ - #leading 1 - 1, - #leading b - b: 2 - #trailing b - ] # trailing outside - """) - - assert {:__block__, keyword_list_meta, - [ - [ - {:__block__, one_key_meta, [1]}, - { - {:__block__, b_key_meta, [:b]}, - {:__block__, b_value_meta, [2]} - } - ] - ]} = quoted - - assert [%{line: 2, text: "#leading 1"}] = one_key_meta[:leading_comments] - assert [%{line: 4, text: "#leading b"}] = b_key_meta[:leading_comments] - assert [%{line: 6, text: "#trailing b"}] = b_value_meta[:trailing_comments] - assert [%{line: 7, text: "# trailing outside"}] = keyword_list_meta[:trailing_comments] - end - - # Tuples - - test "merges comments in n-tuple" do - quoted = - parse_string!(""" - { - #leading 1 - 1, - #leading 2 - 2, - 3 - #trailing 3 - } # trailing outside - """) - - assert {:{}, tuple_meta, - [ - {:__block__, one_meta, [1]}, - {:__block__, two_meta, [2]}, - {:__block__, three_meta, [3]} - ]} = quoted - - assert [%{line: 2, text: "#leading 1"}] = one_meta[:leading_comments] - assert [%{line: 4, text: "#leading 2"}] = two_meta[:leading_comments] - assert [%{line: 7, text: "#trailing 3"}] = three_meta[:trailing_comments] - assert [%{line: 8, text: "# trailing outside"}] = tuple_meta[:trailing_comments] - end - - test "merges comments in 2-tuple" do - quoted = - parse_string!(""" - { - #leading 1 - 1, - #leading 2 - 2 - #trailing 2 - } # trailing outside - """) - - assert {:__block__, tuple_meta, - [ - { - {:__block__, one_meta, [1]}, - {:__block__, two_meta, [2]} - } - ]} = quoted - - assert [%{line: 2, text: "#leading 1"}] = one_meta[:leading_comments] - assert [%{line: 4, text: "#leading 2"}] = two_meta[:leading_comments] - assert [%{line: 6, text: "#trailing 2"}] = two_meta[:trailing_comments] - assert [%{line: 7, text: "# trailing outside"}] = tuple_meta[:trailing_comments] - end - - test "merges inner comments in empty tuple" do - quoted = - parse_string!(""" - { - # inner 1 - # inner 2 - } # trailing outside - """) - - assert {:{}, tuple_meta, []} = quoted - - assert [%{line: 2, text: "# inner 1"}, %{line: 3, text: "# inner 2"}] = - tuple_meta[:inner_comments] - - assert [%{line: 4, text: "# trailing outside"}] = tuple_meta[:trailing_comments] - end - - # Maps - - test "merges comments in maps" do - quoted = - parse_string!(""" - %{ - #leading 1 - 1 => 1, - #leading 2 - 2 => 2, - 3 => 3 - #trailing 3 - } # trailing outside - """) - - assert {:%{}, map_meta, - [ - { - {:__block__, one_key_meta, [1]}, - {:__block__, _, [1]} - }, - { - {:__block__, two_key_meta, [2]}, - {:__block__, _, [2]} - }, - { - {:__block__, _, [3]}, - {:__block__, three_value_meta, [3]} - } - ]} = quoted - - assert [%{line: 2, text: "#leading 1"}] = one_key_meta[:leading_comments] - assert [%{line: 4, text: "#leading 2"}] = two_key_meta[:leading_comments] - assert [%{line: 7, text: "#trailing 3"}] = three_value_meta[:trailing_comments] - assert [%{line: 8, text: "# trailing outside"}] = map_meta[:trailing_comments] - end - - test "merges inner comments in empty maps" do - quoted = - parse_string!(""" - %{ - # inner 1 - # inner 2 - } # trailing outside - """) - - assert {:%{}, map_meta, []} = quoted - - assert [%{line: 2, text: "# inner 1"}, %{line: 3, text: "# inner 2"}] = - map_meta[:inner_comments] - - assert [%{line: 4, text: "# trailing outside"}] = map_meta[:trailing_comments] - end - end - - test "handles the presence of unquote_splicing" do - quoted = - parse_string!(""" - %{ - # leading baz - :baz => :bat, - :quux => :quuz, - # leading unquote splicing - unquote_splicing(foo: :bar) - # trailing unquote splicing - } - """) - - assert {:%{}, _, - [ - {{:__block__, baz_key_meta, [:baz]}, {:__block__, _, [:bat]}}, - {{:__block__, _, [:quux]}, {:__block__, _, [:quuz]}}, - {:unquote_splicing, unquote_splicing_meta, _} - ]} = quoted - - assert [%{line: 2, text: "# leading baz"}] = baz_key_meta[:leading_comments] - - assert [%{line: 5, text: "# leading unquote splicing"}] = - unquote_splicing_meta[:leading_comments] - - assert [%{line: 7, text: "# trailing unquote splicing"}] = - unquote_splicing_meta[:trailing_comments] - end - - # Structs - - test "merges comments in structs" do - quoted = - parse_string!(""" - %SomeStruct{ - #leading 1 - a: 1, - #leading 2 - b: 2, - c: 3 - #trailing 3 - } # trailing outside - """) - - assert {:%, struct_meta, - [ - {:__aliases__, _, [:SomeStruct]}, - {:%{}, _, - [ - {{:__block__, a_key_meta, [:a]}, {:__block__, _, [1]}}, - {{:__block__, b_key_meta, [:b]}, {:__block__, _, [2]}}, - {{:__block__, _, [:c]}, {:__block__, c_value_meta, [3]}} - ]} - ]} = quoted - - assert [%{line: 2, text: "#leading 1"}] = a_key_meta[:leading_comments] - assert [%{line: 4, text: "#leading 2"}] = b_key_meta[:leading_comments] - assert [%{line: 7, text: "#trailing 3"}] = c_value_meta[:trailing_comments] - assert [%{line: 8, text: "# trailing outside"}] = struct_meta[:trailing_comments] - end - - test "merges inner comments in structs" do - quoted = - parse_string!(""" - %SomeStruct{ - # inner 1 - # inner 2 - } # trailing outside - """) - - assert {:%, struct_meta, - [ - {:__aliases__, _, [:SomeStruct]}, - {:%{}, args_meta, []} - ]} = quoted - - assert [%{line: 2, text: "# inner 1"}, %{line: 3, text: "# inner 2"}] = - args_meta[:inner_comments] - - assert [%{line: 4, text: "# trailing outside"}] = struct_meta[:trailing_comments] - end -end From 67fb06e5aa047d17b993a4515740afa5ac945f84 Mon Sep 17 00:00:00 2001 From: doorgan Date: Mon, 31 Mar 2025 14:26:16 -0300 Subject: [PATCH 4/7] Cleanups --- lib/elixir/lib/code/comments.ex | 57 ++++++----- .../test/elixir/code/ast_comments_test.exs | 95 ++++++++++++++++++- 2 files changed, 126 insertions(+), 26 deletions(-) diff --git a/lib/elixir/lib/code/comments.ex b/lib/elixir/lib/code/comments.ex index 9ce3eb8839..d4e66a6449 100644 --- a/lib/elixir/lib/code/comments.ex +++ b/lib/elixir/lib/code/comments.ex @@ -269,7 +269,7 @@ defmodule Code.Comments do end_line = get_end_line(quoted, start_line) {trailing_comments, comments} = - Enum.split_with(comments, &(&1.line > start_line and &1.line < end_line)) + split_trailing_comments(comments, start_line, end_line) quoted = append_comments(quoted, :inner_comments, trailing_comments) @@ -281,7 +281,7 @@ defmodule Code.Comments do end_line = get_end_line(quoted, start_line) {trailing_comments, comments} = - Enum.split_with(comments, &(&1.line > start_line and &1.line < end_line)) + split_trailing_comments(comments, start_line, end_line) last_value = append_comments(last_value, :trailing_comments, trailing_comments) @@ -295,7 +295,7 @@ defmodule Code.Comments do end_line = get_end_line(quoted, start_line) {trailing_comments, comments} = - Enum.split_with(comments, &(&1.line > start_line and &1.line < end_line)) + split_trailing_comments(comments, start_line, end_line) unquote_splicing = append_comments(unquote_splicing, :trailing_comments, trailing_comments) @@ -316,7 +316,7 @@ defmodule Code.Comments do end_line = get_end_line(quoted, start_line) {trailing_comments, comments} = - Enum.split_with(comments, &(&1.line > start_line and &1.line < end_line)) + split_trailing_comments(comments, start_line, end_line) value = append_comments(value, :trailing_comments, trailing_comments) @@ -342,7 +342,7 @@ defmodule Code.Comments do end_line = get_end_line(quoted, start_line) {trailing_comments, comments} = - Enum.split_with(comments, &(&1.line > start_line and &1.line < end_line)) + split_trailing_comments(comments, start_line, end_line) right = append_comments(right, :trailing_comments, trailing_comments) @@ -366,7 +366,7 @@ defmodule Code.Comments do line = get_line(call) {trailing_comments, comments} = - Enum.split_with(state.comments, &(&1.line > line and &1.line < end_line)) + split_trailing_comments(state.comments, line, end_line) call = append_comments(call, :trailing_comments, trailing_comments) @@ -380,7 +380,7 @@ defmodule Code.Comments do case left do [] -> {leading_comments, comments} = - Enum.split_with(comments, &(&1.line > block_start and &1.line < start_line)) + split_leading_comments(comments, block_start, start_line) quoted = append_comments(quoted, :leading_comments, leading_comments) @@ -401,11 +401,11 @@ defmodule Code.Comments do with true <- is_atom(form), <<"sigil_", _name::binary>> <- Atom.to_string(form), true <- not is_nil(meta) do - [content, modifiers] = args + [content | modifiers] = args {content, state} = merge_mixed_comments(content, state) - quoted = put_args(quoted, [content, modifiers]) + quoted = put_args(quoted, [content | modifiers]) {quoted, state} else @@ -461,7 +461,7 @@ defmodule Code.Comments do line = get_line(last_value) {trailing_comments, comments} = - Enum.split_with(comments, &(&1.line > line and &1.line < end_line)) + split_trailing_comments(comments, line, end_line) last_value = append_comments(last_value, :trailing_comments, trailing_comments) @@ -488,7 +488,7 @@ defmodule Code.Comments do start_line = get_line(last_block_arg) {trailing_comments, comments} = - Enum.split_with(comments, &(&1.line > start_line and &1.line < end_line)) + split_trailing_comments(comments, start_line, end_line) last_block_arg = append_comments(last_block_arg, :trailing_comments, trailing_comments) @@ -510,7 +510,7 @@ defmodule Code.Comments do line = get_end_line(last_arg, get_line(last_arg)) {trailing_comments, comments} = - Enum.split_with(comments, &(&1.line > line and &1.line < end_line)) + split_trailing_comments(comments, line, end_line) last_arg = append_comments(last_arg, :trailing_comments, trailing_comments) @@ -523,7 +523,7 @@ defmodule Code.Comments do nil -> {trailing_comments, comments} = - Enum.split_with(comments, &(&1.line > start_line and &1.line < end_line)) + split_trailing_comments(comments, start_line, end_line) quoted = append_comments(quoted, :inner_comments, trailing_comments) {quoted, comments} @@ -544,10 +544,10 @@ defmodule Code.Comments do value_line = get_line(value) {leading_comments, comments} = - Enum.split_with(state.comments, &(&1.line > start_line and &1.line <= value_line)) + split_leading_comments(state.comments, start_line, value_line) {trailing_comments, comments} = - Enum.split_with(comments, &(&1.line > value_line and &1.line < end_line)) + split_trailing_comments(comments, value_line, end_line) value = put_leading_comments(value, leading_comments) value = put_trailing_comments(value, trailing_comments) @@ -566,10 +566,10 @@ defmodule Code.Comments do value_line = get_line(value) {leading_comments, comments} = - Enum.split_with(state.comments, &(&1.line > start_line and &1.line <= value_line)) + split_leading_comments(state.comments, start_line, value_line) {trailing_comments, comments} = - Enum.split_with(comments, &(&1.line > value_line and &1.line < end_line)) + split_trailing_comments(comments, value_line, end_line) value = put_leading_comments(value, leading_comments) value = put_trailing_comments(value, trailing_comments) @@ -590,7 +590,7 @@ defmodule Code.Comments do end_line = get_end_line(quoted, start_line) {trailing_comments, comments} = - Enum.split_with(comments, &(&1.line > start_line and &1.line < end_line)) + split_trailing_comments(comments, start_line, end_line) last_value = append_comments(last_value, :trailing_comments, trailing_comments) @@ -603,7 +603,7 @@ defmodule Code.Comments do end_line = get_end_line(quoted, start_line) {trailing_comments, comments} = - Enum.split_with(comments, &(&1.line > start_line and &1.line < end_line)) + split_trailing_comments(comments, start_line, end_line) unquote_splicing = append_comments(unquote_splicing, :trailing_comments, trailing_comments) @@ -654,7 +654,7 @@ defmodule Code.Comments do case last_arg do nil -> {trailing_comments, comments} = - Enum.split_with(comments, &(&1.line > block_start and &1.line < block_end)) + split_trailing_comments(comments, block_start, block_end) block_args = append_comments(block_args, :inner_comments, trailing_comments) @@ -736,7 +736,7 @@ defmodule Code.Comments do stab_line = get_line(stab) {leading_comments, comments} = - Enum.split_with(comments, &(&1.line > block_start and &1.line < stab_line)) + split_leading_comments(comments, block_start, stab_line) stab = append_comments(stab, :leading_comments, leading_comments) @@ -753,7 +753,7 @@ defmodule Code.Comments do call -> {trailing_comments, comments} = - Enum.split_with(comments, &(&1.line > start_line and &1.line < end_line)) + split_trailing_comments(comments, start_line, end_line) call = append_comments(call, :trailing_comments, trailing_comments) @@ -775,7 +775,7 @@ defmodule Code.Comments do case last_arg do nil -> {trailing_comments, comments} = - Enum.split_with(comments, &(&1.line > block_start and &1.line < block_end)) + split_trailing_comments(comments, block_start, block_end) trailing_comments = Enum.sort_by(trailing_comments, & &1.line) @@ -810,13 +810,14 @@ defmodule Code.Comments do line = case last_arg do [] -> block_start + [{_key, value} | _] -> get_line(value) [first | _] -> get_line(first) {_, _, _} -> get_line(last_arg) _ -> block_start end {trailing_comments, comments} = - Enum.split_with(comments, &(&1.line > line and &1.line < block_end)) + split_trailing_comments(comments, line, block_end) last_arg = append_comments(last_arg, :trailing_comments, trailing_comments) @@ -938,4 +939,12 @@ defmodule Code.Comments do _ -> false end) end + + defp split_leading_comments(comments, min, max) do + Enum.split_with(comments, &(&1.line > min and &1.line <= max)) + end + + defp split_trailing_comments(comments, min, max) do + Enum.split_with(comments, &(&1.line > min and &1.line < max)) + end end diff --git a/lib/elixir/test/elixir/code/ast_comments_test.exs b/lib/elixir/test/elixir/code/ast_comments_test.exs index c1a833942f..fda6a4635d 100644 --- a/lib/elixir/test/elixir/code/ast_comments_test.exs +++ b/lib/elixir/test/elixir/code/ast_comments_test.exs @@ -4,7 +4,11 @@ defmodule Code.AstCommentsTest do use ExUnit.Case, async: true def parse_string!(string) do - Code.string_to_quoted!(string, include_comments: true, literal_encoder: &{:ok, {:__block__, &2, [&1]}}, emit_warnings: false) + Code.string_to_quoted!(string, + include_comments: true, + literal_encoder: &{:ok, {:__block__, &2, [&1]}}, + emit_warnings: false + ) end describe "merge_comments/2" do @@ -210,7 +214,6 @@ defmodule Code.AstCommentsTest do #trailing 3 ] # trailing outside """) - |> IO.inspect() assert {:__block__, list_meta, [ @@ -671,5 +674,93 @@ defmodule Code.AstCommentsTest do assert [%{line: 2, text: "# leading"}] = stab_meta[:leading_comments] end + + # String Interpolations + + test "merges comments in interpolations" do + quoted = + parse_string!(~S""" + # leading + "Hello #{world}" + # trailing + """) + + assert {:<<>>, meta, + [ + "Hello ", + {:"::", _, + [{{:., _, [Kernel, :to_string]}, _, [{:world, _, _}]}, {:binary, _, _}]} + ]} = quoted + + assert [%{line: 1, text: "# leading"}] = meta[:leading_comments] + assert [%{line: 3, text: "# trailing"}] = meta[:trailing_comments] + end + + test "merges comments in interpolated strings" do + quoted = + parse_string!(~S""" + # leading + "Hello #{ + # leading world + world + # trailing world + }" + # trailing + """) + + assert {:<<>>, meta, + [ + "Hello ", + {:"::", _, + [{{:., _, [Kernel, :to_string]}, _, [{:world, world_meta, _}]}, {:binary, _, _}]} + ]} = quoted + + assert [%{line: 1, text: "# leading"}] = meta[:leading_comments] + assert [%{line: 3, text: "# leading world"}] = world_meta[:leading_comments] + assert [%{line: 5, text: "# trailing world"}] = world_meta[:trailing_comments] + assert [%{line: 7, text: "# trailing"}] = meta[:trailing_comments] + end + + # List interpolations + + test "merges comments in list interpolations" do + quoted = + parse_string!(~S""" + # leading + 'Hello #{world}' + # trailing + """) + + assert {{:., _, [List, :to_charlist]}, meta, + [ + ["Hello ", {{:., _, [Kernel, :to_string]}, _, [{:world, _, _}]}] + ]} = quoted + + assert [%{line: 1, text: "# leading"}] = meta[:leading_comments] + assert [%{line: 3, text: "# trailing"}] = meta[:trailing_comments] + end + + test "merges comments in list interpolations with comments" do + quoted = + parse_string!(~S""" + # leading + 'Hello #{ + # leading world + world + # trailing world + }' + # trailing + """) + + assert {{:., _, [List, :to_charlist]}, meta, + [ + ["Hello ", {{:., _, [Kernel, :to_string]}, _, [{:world, world_meta, _}]}] + ]} = quoted + + assert [%{line: 1, text: "# leading"}] = meta[:leading_comments] + assert [%{line: 3, text: "# leading world"}] = world_meta[:leading_comments] + assert [%{line: 5, text: "# trailing world"}] = world_meta[:trailing_comments] + assert [%{line: 7, text: "# trailing"}] = meta[:trailing_comments] + end end end From 05afe408b2fa60f8ed358c3134c5a36635ddc736 Mon Sep 17 00:00:00 2001 From: doorgan Date: Mon, 31 Mar 2025 16:37:00 -0300 Subject: [PATCH 5/7] Fix getting start line for last arg --- lib/elixir/lib/code/comments.ex | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/elixir/lib/code/comments.ex b/lib/elixir/lib/code/comments.ex index d4e66a6449..ee415c0f7e 100644 --- a/lib/elixir/lib/code/comments.ex +++ b/lib/elixir/lib/code/comments.ex @@ -810,12 +810,13 @@ defmodule Code.Comments do line = case last_arg do [] -> block_start - [{_key, value} | _] -> get_line(value) - [first | _] -> get_line(first) - {_, _, _} -> get_line(last_arg) + [{_key, value} | _] -> get_end_line(value, get_line(value)) + [first | _] -> get_end_line(first, get_line(first)) + {_, _, _} -> get_end_line(last_arg, get_line(last_arg)) _ -> block_start end + {trailing_comments, comments} = split_trailing_comments(comments, line, block_end) From c0e4f8bc925463cefd7508a9008deb0bdcec97f6 Mon Sep 17 00:00:00 2001 From: doorgan Date: Mon, 31 Mar 2025 17:01:57 -0300 Subject: [PATCH 6/7] Fix handling of stab comments --- lib/elixir/lib/code/comments.ex | 15 --------------- lib/elixir/lib/code/formatter.ex | 8 ++++++-- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/lib/elixir/lib/code/comments.ex b/lib/elixir/lib/code/comments.ex index ee415c0f7e..9ff8ffbfb4 100644 --- a/lib/elixir/lib/code/comments.ex +++ b/lib/elixir/lib/code/comments.ex @@ -354,7 +354,6 @@ defmodule Code.Comments do defp merge_mixed_comments({:->, _, [left, right]} = quoted, state) do start_line = get_line(right) end_line = get_end_line({:__block__, state.parent_meta, [quoted]}, start_line) - block_start = get_line({:__block__, state.parent_meta, [quoted]}) {right, comments} = case right do @@ -376,20 +375,6 @@ defmodule Code.Comments do end end - {quoted, comments} = - case left do - [] -> - {leading_comments, comments} = - split_leading_comments(comments, block_start, start_line) - - quoted = append_comments(quoted, :leading_comments, leading_comments) - - {quoted, comments} - - _ -> - {quoted, comments} - end - quoted = put_args(quoted, [left, right]) {quoted, %{state | comments: comments}} diff --git a/lib/elixir/lib/code/formatter.ex b/lib/elixir/lib/code/formatter.ex index 6131e320d8..eab5e7e933 100644 --- a/lib/elixir/lib/code/formatter.ex +++ b/lib/elixir/lib/code/formatter.ex @@ -1891,9 +1891,13 @@ defmodule Code.Formatter do comments = merge_algebra_with_comments(comments, @empty) # If there are any comments before the ->, we hoist them up above the fn - doc = case comments do + doc = + case comments do [] -> doc - [comments] -> line(comments, doc) + [comment] -> line(comment, doc) + comments -> + comments_doc = comments |> Enum.reduce(&line(&2, &1)) |> force_unfit() + line(comments_doc, doc) end {doc, state} From c97ee1fd4d87723c93a6f2e44caa0a4f497d079a Mon Sep 17 00:00:00 2001 From: doorgan Date: Mon, 31 Mar 2025 22:59:33 -0300 Subject: [PATCH 7/7] Format binary operators comments --- lib/elixir/lib/code/formatter.ex | 93 +++++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 2 deletions(-) diff --git a/lib/elixir/lib/code/formatter.ex b/lib/elixir/lib/code/formatter.ex index eab5e7e933..e36425a9f8 100644 --- a/lib/elixir/lib/code/formatter.ex +++ b/lib/elixir/lib/code/formatter.ex @@ -807,6 +807,16 @@ defmodule Code.Formatter do left_context = left_op_context(context) right_context = right_op_context(context) + {comments, [left_arg, right_arg]} = pop_binary_op_chain_comments(op, [left_arg, right_arg], []) + comments = Enum.sort_by(comments, &(&1.line)) + comments_docs = + Enum.map(comments, fn comment -> + comment = format_comment(comment) + {comment.text, @empty, 1} + end) + + comments_docs = merge_algebra_with_comments(comments_docs, @empty) + {left, state} = binary_operand_to_algebra(left_arg, left_context, state, op, op_info, :left, 2) @@ -826,7 +836,6 @@ defmodule Code.Formatter do next_break_fits? = op in @next_break_fits_operators and next_break_fits?(right_arg, state) and not eol? - {" " <> op_string, with_next_break_fits(next_break_fits?, right, fn right -> right = nest(concat(break(), right), nesting, :break) @@ -835,10 +844,62 @@ defmodule Code.Formatter do end op_doc = color_doc(op_string, :operator, state.inspect_opts) - doc = concat(concat(group(left), op_doc), group(right)) + doc = concat(doc = concat(group(left), op_doc), group(right)) + + doc = + case comments_docs do + [] -> doc + [line] -> line(line, doc) + lines -> line(lines |> Enum.reduce(&line(&2, &1)) |> force_unfit(), doc) + end {doc, state} end + defp pop_binary_op_chain_comments(op, [{_, left_meta, _} = left, right], acc) do + left_leading = List.wrap(left_meta[:leading_comments]) + left_trailing = List.wrap(left_meta[:trailing_comments]) + + left = Macro.update_meta(left, &Keyword.drop(&1, [:leading_comments, :trailing_comments])) + + acc = Enum.concat([left_leading, left_trailing, acc]) + + {_assoc, prec} = augmented_binary_op(op) + + with {right_op, right_meta, right_args} <- right, + true <- right_op not in @pipeline_operators, + true <- right_op not in @right_new_line_before_binary_operators, + {_, right_prec} <- augmented_binary_op(right_op) do + {acc, right_args} = pop_binary_op_chain_comments(right_op, right_args, acc) + + right = {right_op, right_meta, right_args} + + {acc, [left, right]} + else + _ -> + {acc, right} = + case right do + {_, right_meta, _} -> + right_leading = List.wrap(right_meta[:leading_comments]) + right_trailing = List.wrap(right_meta[:trailing_comments]) + + right = Macro.update_meta(right, &Keyword.drop(&1, [:leading_comments, :trailing_comments])) + + acc = Enum.concat([right_leading, right_trailing, acc]) + + {acc, right} + + _ -> + {acc, right} + end + + {acc, [left, right]} + end + end + + defp pop_binary_op_chain_comments(_, args, acc) do + {acc, args} + end + # TODO: We can remove this workaround once we remove # ?rearrange_uop from the parser on v2.0. # (! left) in right @@ -1624,6 +1685,34 @@ defmodule Code.Formatter do fun = "ed_to_algebra(&1, :parens_arg, &2) {left_doc, state} = fun.(left, state) + before_cons_comments = + case left do + {_, meta, _} -> + List.wrap(meta[:trailing_comments]) + + _ -> + [] + end + + right = + case right do + {_, _, _} -> + Macro.update_meta(right, fn meta -> + Keyword.update(meta, :leading_comments, before_cons_comments, &(before_cons_comments ++ &1)) + end) + + [{{_, _, _} = key, value} | rest] -> + key = + Macro.update_meta(key, fn meta -> + Keyword.update(meta, :leading_comments, before_cons_comments, &(before_cons_comments ++ &1)) + end) + + [{key, value} | rest] + + _ -> + right + end + {right_doc, _join, state} = args_to_algebra_with_comments(right, meta, :none, join, state, fun)