Skip to content

Commit 11e1859

Browse files
aycabtaaycabta
committed
Merge branch 'master' into fix-handling-hash-rocket-after-symbol
2 parents 98c7020 + d3f8522 commit 11e1859

File tree

5 files changed

+302
-45
lines changed

5 files changed

+302
-45
lines changed

‎lib/rdoc/ruby_lex.rb

Lines changed: 121 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -396,7 +396,9 @@ def token
396396
"r" => "/",
397397
"w" => "]",
398398
"W" => "]",
399-
"s" => ":"
399+
"s" => ":",
400+
"i" => "]",
401+
"I" => "]"
400402
}
401403

402404
PERCENT_PAREN = {
@@ -479,25 +481,50 @@ def lex_init()
479481
Token(TkNL)
480482
end
481483

484+
@OP.def_rules("=") do
485+
|op, io|
486+
case @lex_state
487+
when :EXPR_FNAME, :EXPR_DOT
488+
@lex_state = :EXPR_ARG
489+
else
490+
@lex_state = :EXPR_BEG
491+
end
492+
Token(op)
493+
end
494+
482495
@OP.def_rules("*", "**",
483-
"=", "==", "===",
496+
"==", "===",
484497
"=~", "<=>",
485498
"<", "<=",
486499
">", ">=", ">>", "=>") do
487500
|op, io|
488501
case @lex_state
489502
when :EXPR_FNAME, :EXPR_DOT
503+
tk = Token(TkId, op)
490504
@lex_state = :EXPR_ARG
491505
else
506+
tk = Token(op)
492507
@lex_state = :EXPR_BEG
493508
end
509+
tk
510+
end
511+
512+
@OP.def_rules("->") do
513+
|op, io|
514+
@lex_state = :EXPR_ENDFN
494515
Token(op)
495516
end
496517

497518
@OP.def_rules("!", "!=", "!~") do
498519
|op, io|
499-
@lex_state = :EXPR_BEG
500-
Token(op)
520+
case @lex_state
521+
when :EXPR_FNAME, :EXPR_DOT
522+
@lex_state = :EXPR_ARG
523+
Token(TkId, op)
524+
else
525+
@lex_state = :EXPR_BEG
526+
Token(op)
527+
end
501528
end
502529

503530
@OP.def_rules("<<") do
@@ -511,11 +538,12 @@ def lex_init()
511538
end
512539
end
513540
unless tk
514-
tk = Token(op)
515541
case @lex_state
516542
when :EXPR_FNAME, :EXPR_DOT
543+
tk = Token(TkId, op)
517544
@lex_state = :EXPR_ARG
518545
else
546+
tk = Token(op)
519547
@lex_state = :EXPR_BEG
520548
end
521549
end
@@ -529,9 +557,9 @@ def lex_init()
529557

530558
@OP.def_rules("`") do
531559
|op, io|
532-
if @lex_state == :EXPR_FNAME
533-
@lex_state = :EXPR_END
534-
Token(op)
560+
if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
561+
@lex_state = :EXPR_ARG
562+
Token(TkId, op)
535563
else
536564
identify_string(op)
537565
end
@@ -555,12 +583,25 @@ def lex_init()
555583
end
556584
end
557585

558-
@OP.def_rules("&", "&&", "|", "||") do
586+
@OP.def_rules("&&", "||") do
559587
|op, io|
560588
@lex_state = :EXPR_BEG
561589
Token(op)
562590
end
563591

592+
@OP.def_rules("&", "|") do
593+
|op, io|
594+
case @lex_state
595+
when :EXPR_FNAME, :EXPR_DOT
596+
tk = Token(TkId, op)
597+
@lex_state = :EXPR_ARG
598+
else
599+
tk = Token(op)
600+
@lex_state = :EXPR_BEG
601+
end
602+
tk
603+
end
604+
564605
@OP.def_rules("+=", "-=", "*=", "**=",
565606
"&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do
566607
|op, io|
@@ -572,19 +613,22 @@ def lex_init()
572613
@OP.def_rule("+@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
573614
|op, io|
574615
@lex_state = :EXPR_ARG
575-
Token(op)
616+
Token(TkId, op)
576617
end
577618

578619
@OP.def_rule("-@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
579620
|op, io|
580621
@lex_state = :EXPR_ARG
581-
Token(op)
622+
Token(TkId, op)
582623
end
583624

584625
@OP.def_rules("+", "-") do
585626
|op, io|
586627
catch(:RET) do
587-
if @lex_state == :EXPR_ARG
628+
if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
629+
tk = Token(TkId, op)
630+
@lex_state = :EXPR_ARG
631+
elsif @lex_state == :EXPR_ARG
588632
if @space_seen and peek(0) =~ /[0-9]/
589633
throw :RET, identify_number(op)
590634
else
@@ -595,7 +639,8 @@ def lex_init()
595639
else
596640
@lex_state = :EXPR_BEG
597641
end
598-
Token(op)
642+
tk = Token(op) unless tk
643+
tk
599644
end
600645
end
601646

@@ -655,7 +700,10 @@ def lex_int2
655700

656701
@OP.def_rule("/") do
657702
|op, io|
658-
if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
703+
if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
704+
@lex_state = :EXPR_ARG
705+
Token(TkId, op)
706+
elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
659707
identify_string(op)
660708
elsif peek(0) == '='
661709
getc
@@ -671,8 +719,15 @@ def lex_int2
671719

672720
@OP.def_rules("^") do
673721
|op, io|
674-
@lex_state = :EXPR_BEG
675-
Token("^")
722+
case @lex_state
723+
when :EXPR_FNAME, :EXPR_DOT
724+
tk = Token(TkId, op)
725+
@lex_state = :EXPR_ARG
726+
else
727+
tk = Token(op)
728+
@lex_state = :EXPR_BEG
729+
end
730+
tk
676731
end
677732

678733
# @OP.def_rules("^=") do
@@ -699,8 +754,14 @@ def lex_int2
699754

700755
@OP.def_rule("~") do
701756
|op, io|
702-
@lex_state = :EXPR_BEG
703-
Token("~")
757+
case @lex_state
758+
when :EXPR_FNAME, :EXPR_DOT
759+
@lex_state = :EXPR_ARG
760+
Token(TkId, op)
761+
else
762+
@lex_state = :EXPR_BEG
763+
Token(op)
764+
end
704765
end
705766

706767
@OP.def_rule("~@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
@@ -726,17 +787,18 @@ def lex_int2
726787
@OP.def_rule("[]", proc{|op, io| @lex_state == :EXPR_FNAME}) do
727788
|op, io|
728789
@lex_state = :EXPR_ARG
729-
Token("[]")
790+
Token(TkId, op)
730791
end
731792

732793
@OP.def_rule("[]=", proc{|op, io| @lex_state == :EXPR_FNAME}) do
733794
|op, io|
734795
@lex_state = :EXPR_ARG
735-
Token("[]=")
796+
Token(TkId, op)
736797
end
737798

738799
@OP.def_rule("[") do
739800
|op, io|
801+
text = nil
740802
@indent += 1
741803
if @lex_state == :EXPR_FNAME
742804
tk_c = TkfLBRACK
@@ -745,13 +807,25 @@ def lex_int2
745807
tk_c = TkLBRACK
746808
elsif @lex_state == :EXPR_ARG && @space_seen
747809
tk_c = TkLBRACK
810+
elsif @lex_state == :EXPR_DOT
811+
if peek(0) == "]"
812+
tk_c = TkIDENTIFIER
813+
getc
814+
if peek(0) == "="
815+
text = "[]="
816+
else
817+
text = "[]"
818+
end
819+
else
820+
tk_c = TkOp
821+
end
748822
else
749823
tk_c = TkfLBRACK
750824
end
751825
@lex_state = :EXPR_BEG
752826
end
753827
@indent_stack.push tk_c
754-
Token(tk_c)
828+
Token(tk_c, text)
755829
end
756830

757831
@OP.def_rule("{") do
@@ -769,19 +843,19 @@ def lex_int2
769843

770844
@OP.def_rule('\\') do
771845
|op, io|
772-
if getc == "\n"
846+
if peek(0) == "\n"
773847
@space_seen = true
774848
@continue = true
775-
Token(TkSPACE)
776-
else
777-
ungetc
778-
Token("\\")
779849
end
850+
Token("\\")
780851
end
781852

782853
@OP.def_rule('%') do
783854
|op, io|
784-
if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
855+
if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
856+
@lex_state = :EXPR_ARG
857+
Token(TkId, op)
858+
elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
785859
identify_quotation
786860
elsif peek(0) == '='
787861
getc
@@ -948,7 +1022,12 @@ def identify_identifier
9481022
@indent_stack.push token_c
9491023
end
9501024
else
951-
token_c = TkIDENTIFIER
1025+
if peek(0) == ':' and !peek_match?(/^::/)
1026+
token.concat getc
1027+
token_c = TkSYMBOL
1028+
else
1029+
token_c = TkIDENTIFIER
1030+
end
9521031
end
9531032

9541033
elsif DEINDENT_CLAUSE.include?(token)
@@ -977,11 +1056,20 @@ def identify_identifier
9771056
end
9781057

9791058
if token[0, 1] =~ /[A-Z]/
980-
return Token(TkCONSTANT, token)
1059+
if token[-1] =~ /[!?]/
1060+
return Token(TkIDENTIFIER, token)
1061+
else
1062+
return Token(TkCONSTANT, token)
1063+
end
9811064
elsif token[token.size - 1, 1] =~ /[!?]/
9821065
return Token(TkFID, token)
9831066
else
984-
return Token(TkIDENTIFIER, token)
1067+
if peek(0) == ':' and !peek_match?(/^::/)
1068+
token.concat getc
1069+
return Token(TkSYMBOL, token)
1070+
else
1071+
return Token(TkIDENTIFIER, token)
1072+
end
9851073
end
9861074
end
9871075

@@ -1211,7 +1299,7 @@ def identify_string(ltype, quoted = ltype, type = nil)
12111299
str = if ltype == quoted and %w[" ' /].include? ltype then
12121300
ltype.dup
12131301
else
1214-
"%#{type or PERCENT_LTYPE.key ltype}#{PERCENT_PAREN_REV[quoted]||quoted}"
1302+
"%#{type}#{PERCENT_PAREN_REV[quoted]||quoted}"
12151303
end
12161304

12171305
subtype = nil
@@ -1225,8 +1313,8 @@ def identify_string(ltype, quoted = ltype, type = nil)
12251313
break
12261314
elsif @ltype != "'" && @ltype != "]" && @ltype != ":" and ch == "#"
12271315
ch = getc
1228-
subtype = true
12291316
if ch == "{" then
1317+
subtype = true
12301318
str << ch << skip_inner_expression
12311319
next
12321320
else
@@ -1235,7 +1323,7 @@ def identify_string(ltype, quoted = ltype, type = nil)
12351323
elsif ch == '\\'
12361324
if %w[' /].include? @ltype then
12371325
case ch = getc
1238-
when "\\", "\n", "'"
1326+
when "\n", "'"
12391327
when @ltype
12401328
str << ch
12411329
else

‎lib/rdoc/ruby_token.rb

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -371,6 +371,7 @@ def Token(token, value = nil)
371371
[:TkCOLON3, TkOp, '::'],
372372
#[:OPASGN, TkOp], # +=, -= etc. #
373373
[:TkASSOC, TkOp, "=>"],
374+
[:TkLAMBDA, TkOp, "->"],
374375
[:TkQUESTION, TkOp, "?"], #?
375376
[:TkCOLON, TkOp, ":"], #:
376377

0 commit comments

Comments
 (0)