Skip to content

Handle redefinable operators as identifier #470

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Aug 8, 2017
Merged

Handle redefinable operators as identifier #470

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e45f7cd
Handle redefinable operators as identifier
aycabta Jul 24, 2017
689c8a2
Add test_accept_verbatim_redefinable_operators
aycabta Jul 29, 2017
59f8d55
Add test_parse_redefinable_methods
aycabta Aug 1, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 77 additions & 21 deletions lib/rdoc/ruby_lex.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -479,25 +479,44 @@ def lex_init()
Token(TkNL)
end

@OP.def_rules("=") do
|op, io|
case @lex_state
when :EXPR_FNAME, :EXPR_DOT
@lex_state = :EXPR_ARG
else
@lex_state = :EXPR_BEG
end
Token(op)
end

@OP.def_rules("*", "**",
"=", "==", "===",
"==", "===",
"=~", "<=>",
"<", "<=",
">", ">=", ">>") do
|op, io|
case @lex_state
when :EXPR_FNAME, :EXPR_DOT
tk = Token(TkId, op)
@lex_state = :EXPR_ARG
else
tk = Token(op)
@lex_state = :EXPR_BEG
end
Token(op)
tk
end

@OP.def_rules("!", "!=", "!~") do
|op, io|
@lex_state = :EXPR_BEG
Token(op)
case @lex_state
when :EXPR_FNAME, :EXPR_DOT
@lex_state = :EXPR_ARG
Token(TkId, op)
else
@lex_state = :EXPR_BEG
Token(op)
end
end

@OP.def_rules("<<") do
Expand All @@ -511,11 +530,12 @@ def lex_init()
end
end
unless tk
tk = Token(op)
case @lex_state
when :EXPR_FNAME, :EXPR_DOT
tk = Token(TkId, op)
@lex_state = :EXPR_ARG
else
tk = Token(op)
@lex_state = :EXPR_BEG
end
end
Expand All @@ -529,9 +549,9 @@ def lex_init()

@OP.def_rules("`") do
|op, io|
if @lex_state == :EXPR_FNAME
@lex_state = :EXPR_END
Token(op)
if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
@lex_state = :EXPR_ARG
Token(TkId, op)
else
identify_string(op)
end
Expand All @@ -555,12 +575,25 @@ def lex_init()
end
end

@OP.def_rules("&", "&&", "|", "||") do
@OP.def_rules("&&", "||") do
|op, io|
@lex_state = :EXPR_BEG
Token(op)
end

@OP.def_rules("&", "|") do
|op, io|
case @lex_state
when :EXPR_FNAME, :EXPR_DOT
tk = Token(TkId, op)
@lex_state = :EXPR_ARG
else
tk = Token(op)
@lex_state = :EXPR_BEG
end
tk
end

@OP.def_rules("+=", "-=", "*=", "**=",
"&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do
|op, io|
Expand All @@ -572,19 +605,22 @@ def lex_init()
@OP.def_rule("+@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
|op, io|
@lex_state = :EXPR_ARG
Token(op)
Token(TkId, op)
end

@OP.def_rule("-@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
|op, io|
@lex_state = :EXPR_ARG
Token(op)
Token(TkId, op)
end

@OP.def_rules("+", "-") do
|op, io|
catch(:RET) do
if @lex_state == :EXPR_ARG
if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
tk = Token(TkId, op)
@lex_state = :EXPR_ARG
elsif @lex_state == :EXPR_ARG
if @space_seen and peek(0) =~ /[0-9]/
throw :RET, identify_number(op)
else
Expand All @@ -595,7 +631,8 @@ def lex_init()
else
@lex_state = :EXPR_BEG
end
Token(op)
tk = Token(op) unless tk
tk
end
end

Expand Down Expand Up @@ -655,7 +692,10 @@ def lex_int2

@OP.def_rule("/") do
|op, io|
if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
@lex_state = :EXPR_ARG
Token(TkId, op)
elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
identify_string(op)
elsif peek(0) == '='
getc
Expand All @@ -671,8 +711,15 @@ def lex_int2

@OP.def_rules("^") do
|op, io|
@lex_state = :EXPR_BEG
Token("^")
case @lex_state
when :EXPR_FNAME, :EXPR_DOT
tk = Token(TkId, op)
@lex_state = :EXPR_ARG
else
tk = Token(op)
@lex_state = :EXPR_BEG
end
tk
end

# @OP.def_rules("^=") do
Expand All @@ -699,8 +746,14 @@ def lex_int2

@OP.def_rule("~") do
|op, io|
@lex_state = :EXPR_BEG
Token("~")
case @lex_state
when :EXPR_FNAME, :EXPR_DOT
@lex_state = :EXPR_ARG
Token(TkId, op)
else
@lex_state = :EXPR_BEG
Token(op)
end
end

@OP.def_rule("~@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
Expand All @@ -726,13 +779,13 @@ def lex_int2
@OP.def_rule("[]", proc{|op, io| @lex_state == :EXPR_FNAME}) do
|op, io|
@lex_state = :EXPR_ARG
Token("[]")
Token(TkId, op)
end

@OP.def_rule("[]=", proc{|op, io| @lex_state == :EXPR_FNAME}) do
|op, io|
@lex_state = :EXPR_ARG
Token("[]=")
Token(TkId, op)
end

@OP.def_rule("[") do
Expand Down Expand Up @@ -781,7 +834,10 @@ def lex_int2

@OP.def_rule('%') do
|op, io|
if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
@lex_state = :EXPR_ARG
Token(TkId, op)
elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
identify_quotation
elsif peek(0) == '='
getc
Expand Down
30 changes: 30 additions & 0 deletions test/test_rdoc_markup_to_html.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -485,6 +485,36 @@ def test_accept_verbatim_ruby
assert_equal expected, @to.res.join
end

def test_accept_verbatim_redefinable_operators
functions = %w[| ^ & <=> == === =~ > >= < <= << >> + - * / % ** ~ +@ -@ [] []= ` ! != !~].map { |redefinable_op|
["def #{redefinable_op}\n", "end\n"]
}.flatten

verb = @RM::Verbatim.new(*functions)

@to.start_accepting
@to.accept_verbatim verb

expected = <<-EXPECTED

<pre class="ruby">
EXPECTED
expected = expected.rstrip

%w[| ^ &amp; &lt;=&gt; == === =~ &gt; &gt;= &lt; &lt;= &lt;&lt; &gt;&gt; + - * / % ** ~ +@ -@ [] []= ` ! != !~].each do |html_escaped_op|
expected += <<-EXPECTED
<span class="ruby-keyword">def</span> <span class="ruby-identifier">#{html_escaped_op}</span>
<span class="ruby-keyword">end</span>
EXPECTED
end

expected += <<-EXPECTED
</pre>
EXPECTED

assert_equal expected, @to.res.join
end

def test_convert_string
assert_equal '&lt;&gt;', @to.convert_string('<>')
end
Expand Down
19 changes: 19 additions & 0 deletions test/test_rdoc_parser_ruby.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1650,6 +1650,25 @@ def test_parse_method
assert_equal stream, foo.token_stream
end

def test_parse_redefinable_methods
klass = RDoc::NormalClass.new 'Foo'
klass.parent = @top_level

comment = RDoc::Comment.new "", @top_level

redefinable_ops = %w[| ^ & <=> == === =~ > >= < <= << >> + - * / % ** ~ +@ -@ [] []= ` ! != !~]
redefinable_ops.each do |redefinable_op|
util_parser "def #{redefinable_op}\nend\n"
tk = @parser.get_tk
@parser.parse_method klass, RDoc::Parser::Ruby::NORMAL, tk, comment
end

klass.method_list.each do |method|
assert_kind_of RDoc::RubyToken::TkId, method.token_stream[5]
assert_includes redefinable_ops, method.token_stream[5].text
end
end

def test_parse_method_bracket
util_parser <<-RUBY
class C
Expand Down