Merge branch 'master' into fix-escape-in-string

Author: aycabta

README.rdoc

@@ -1,10 +1,10 @@
 = \RDoc - Ruby Documentation System
 
-home :: https://github.com/rdoc/rdoc
-rdoc :: https://rdoc.github.io/rdoc
-bugs :: https://github.com/rdoc/rdoc/issues
-build status :: {<img src="https://travis-ci.org/rdoc/rdoc.svg?branch=master" alt="Build Status" />}[https://travis-ci.org/rdoc/rdoc]
-code quality :: {<img src="https://codeclimate.com/github/rdoc/rdoc/badges/gpa.svg" alt="Code Climate">}[https://codeclimate.com/github/rdoc/rdoc]
+home :: https://github.com/ruby/rdoc
+rdoc :: https://ruby.github.io/rdoc
+bugs :: https://github.com/ruby/rdoc/issues
+build status :: {<img src="https://travis-ci.org/ruby/rdoc.svg?branch=master" alt="Build Status" />}[https://travis-ci.org/ruby/rdoc]
+code quality :: {<img src="https://codeclimate.com/github/ruby/rdoc/badges/gpa.svg" alt="Code Climate">}[https://codeclimate.com/github/ruby/rdoc]
 
 == Description
 

lib/rdoc/parser/ruby_tools.rb

@@ -33,26 +33,6 @@ def get_tk
 
     tk = nil if TkEND_OF_SCRIPT === tk
 
-    if TkSYMBEG === tk then
-      set_token_position tk.line_no, tk.char_no
-
-      case tk1 = get_tk
-      when TkId, TkOp, TkSTRING, TkDSTRING, TkSTAR, TkAMPER then
-        if tk1.respond_to?(:name) then
-          tk = Token(TkSYMBOL).set_text(":" + tk1.name)
-        else
-          tk = Token(TkSYMBOL).set_text(":" + tk1.text)
-        end
-
-        # remove the identifier we just read to replace it with a symbol
-        @token_listeners.each do |obj|
-          obj.pop_token
-        end if @token_listeners
-      else
-        tk = tk1
-      end
-    end
-
     # inform any listeners of our shiny new token
     @token_listeners.each do |obj|
       obj.add_token(tk)

lib/rdoc/ruby_lex.rb

@@ -361,6 +361,22 @@ def token
     if @readed_auto_clean_up
       get_readed
     end
+
+    if TkSYMBEG === tk then
+      tk1 = token
+      set_token_position tk.seek, tk.line_no, tk.char_no
+
+      case tk1
+      when TkId, TkOp, TkSTRING, TkDSTRING, TkSTAR, TkAMPER then
+        if tk1.respond_to?(:name) then
+          tk = Token(TkSYMBOL, ":" + tk1.name)
+        else
+          tk = Token(TkSYMBOL, ":" + tk1.text)
+        end
+      else
+        tk = tk1
+      end
+    end
     #      Tracer.off
     tk
   end
@@ -380,7 +396,9 @@ def token
     "r" => "/",
     "w" => "]",
     "W" => "]",
-    "s" => ":"
+    "s" => ":",
+    "i" => "]",
+    "I" => "]"
   }
 
   PERCENT_PAREN = {
@@ -467,7 +485,7 @@ def lex_init()
                   "=", "==", "===",
                   "=~", "<=>",
                   "<", "<=",
-                  ">", ">=", ">>") do
+                  ">", ">=", ">>", "=>") do
       |op, io|
       case @lex_state
       when :EXPR_FNAME, :EXPR_DOT
@@ -721,6 +739,7 @@ def lex_int2
 
     @OP.def_rule("[") do
       |op, io|
+      text = nil
       @indent += 1
       if @lex_state == :EXPR_FNAME
         tk_c = TkfLBRACK
@@ -729,13 +748,25 @@ def lex_int2
           tk_c = TkLBRACK
         elsif @lex_state == :EXPR_ARG && @space_seen
           tk_c = TkLBRACK
+        elsif @lex_state == :EXPR_DOT
+          if peek(0) == "]"
+            tk_c = TkIDENTIFIER
+            getc
+            if peek(0) == "="
+              text = "[]="
+            else
+              text = "[]"
+            end
+          else
+            tk_c = TkOp
+          end
         else
           tk_c = TkfLBRACK
         end
         @lex_state = :EXPR_BEG
       end
       @indent_stack.push tk_c
-      Token(tk_c)
+      Token(tk_c, text)
     end
 
     @OP.def_rule("{") do
@@ -932,7 +963,12 @@ def identify_identifier
                   @indent_stack.push token_c
                 end
               else
-                token_c = TkIDENTIFIER
+                if peek(0) == ':' and !peek_match?(/^::/)
+                  token.concat getc
+                  token_c = TkSYMBOL
+                else
+                  token_c = TkIDENTIFIER
+                end
               end
 
             elsif DEINDENT_CLAUSE.include?(token)
@@ -965,7 +1001,12 @@ def identify_identifier
     elsif token[token.size - 1, 1] =~ /[!?]/
       return Token(TkFID, token)
     else
-      return Token(TkIDENTIFIER, token)
+      if peek(0) == ':' and !peek_match?(/^::/)
+        token.concat getc
+        return Token(TkSYMBOL, token)
+      else
+        return Token(TkIDENTIFIER, token)
+      end
     end
   end
 
@@ -1125,6 +1166,7 @@ def identify_number(op = "")
     type = TkINTEGER
     allow_point = true
     allow_e = true
+    allow_ri = true
     non_digit = false
     while ch = getc
       num << ch
@@ -1154,8 +1196,25 @@ def identify_number(op = "")
           num << getc
         end
         allow_e = false
+        allow_ri = false
         allow_point = false
         non_digit = ch
+      when allow_ri && "r"
+        if non_digit
+          raise Error, "trailing `#{non_digit}' in number"
+        end
+        type = TkRATIONAL
+        if peek(0) == 'i'
+          type = TkIMAGINARY
+          num << getc
+        end
+        break
+      when allow_ri && "i"
+        if non_digit && non_digit != "r"
+          raise Error, "trailing `#{non_digit}' in number"
+        end
+        type = TkIMAGINARY
+        break
       else
         if non_digit
           raise Error, "trailing `#{non_digit}' in number"

lib/rdoc/ruby_token.rb

@@ -26,7 +26,8 @@ module RDoc::RubyToken
     Symbol = Integer
   end
 
-  def set_token_position(line, char)
+  def set_token_position(seek, line, char)
+    @prev_seek = seek
     @prev_line_no = line
     @prev_char_no = char
   end
@@ -331,6 +332,8 @@ def Token(token, value = nil)
 
     [:TkINTEGER,    TkVal],
     [:TkFLOAT,      TkVal],
+    [:TkRATIONAL,   TkVal],
+    [:TkIMAGINARY,  TkVal],
     [:TkSTRING,     TkVal],
     [:TkHEREDOC,    TkVal],
     [:TkXSTRING,    TkVal],
@@ -353,6 +356,7 @@ def Token(token, value = nil)
     [:TkNEQ,        TkOp,   "!="],
     [:TkGEQ,        TkOp,   ">="],
     [:TkLEQ,        TkOp,   "<="],
+    [:TkHASHROCKET, TkOp,   "=>"],
     [:TkANDOP,      TkOp,   "&&"],
     [:TkOROP,       TkOp,   "||"],
     [:TkMATCH,      TkOp,   "=~"],

lib/rdoc/token_stream.rb

@@ -36,9 +36,10 @@ def self.to_html token_stream
               when RDoc::RubyToken::TkIVAR     then 'ruby-ivar'
               when RDoc::RubyToken::TkOp       then 'ruby-operator'
               when RDoc::RubyToken::TkId       then 'ruby-identifier'
+              when RDoc::RubyToken::TkREGEXP   then 'ruby-regexp'
+              when RDoc::RubyToken::TkDREGEXP  then 'ruby-regexp'
               when RDoc::RubyToken::TkNode     then 'ruby-node'
               when RDoc::RubyToken::TkCOMMENT  then 'ruby-comment'
-              when RDoc::RubyToken::TkREGEXP   then 'ruby-regexp'
               when RDoc::RubyToken::TkSTRING   then 'ruby-string'
               when RDoc::RubyToken::TkVal      then 'ruby-value'
               end