Merge pull request #488 from aycabta/fix-creation-of-symbol-token

Fix creation of symbol token

Author: hsbt

lib/rdoc/ruby_lex.rb

@@ -1046,12 +1046,7 @@ def identify_identifier
                   @indent_stack.push token_c
                 end
               else
-                if peek(0) == ':' and !peek_match?(/^::/)
-                  token.concat getc
-                  token_c = TkSYMBOL
-                else
-                  token_c = TkIDENTIFIER
-                end
+                token_c = TkIDENTIFIER
               end
 
             elsif DEINDENT_CLAUSE.include?(token)
@@ -1063,6 +1058,10 @@ def identify_identifier
             @lex_state = :EXPR_END
           end
         end
+        if token_c.ancestors.include?(TkId) and peek(0) == ':' and !peek_match?(/^::/)
+          token.concat getc
+          token_c = TkSYMBOL
+        end
         return Token(token_c, token)
       end
     end
@@ -1081,19 +1080,20 @@ def identify_identifier
 
     if token[0, 1] =~ /[A-Z]/
       if token[-1] =~ /[!?]/
-        return Token(TkIDENTIFIER, token)
+        token_c = TkIDENTIFIER
       else
-        return Token(TkCONSTANT, token)
+        token_c = TkCONSTANT
       end
     elsif token[token.size - 1, 1] =~ /[!?]/
-      return Token(TkFID, token)
+      token_c = TkFID
     else
-      if peek(0) == ':' and !peek_match?(/^::/)
-        token.concat getc
-        return Token(TkSYMBOL, token)
-      else
-        return Token(TkIDENTIFIER, token)
-      end
+      token_c = TkIDENTIFIER
+    end
+    if peek(0) == ':' and !peek_match?(/^::/)
+      token.concat getc
+      return Token(TkSYMBOL, token)
+    else
+      return Token(token_c, token)
     end
   end
 

test/test_rdoc_ruby_lex.rb

@@ -671,6 +671,33 @@ def test_class_tokenize_symbol
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_particular_kind_of_symbols
+    tokens = RDoc::RubyLex.tokenize '{ Thomas: :Thomas, Dave!: :Dave!, undef: :undef }', nil
+
+    expected = [
+      @TK::TkLBRACE.new( 0, 1,  0, "{"),
+      @TK::TkSPACE .new( 1, 1,  1, " "),
+      @TK::TkSYMBOL.new( 2, 1,  2, "Thomas:"),
+      @TK::TkSPACE .new( 9, 1,  9, " "),
+      @TK::TkSYMBOL.new(10, 1, 10, ":Thomas"),
+      @TK::TkCOMMA .new(17, 1, 17, ","),
+      @TK::TkSPACE .new(18, 1, 18, " "),
+      @TK::TkSYMBOL.new(19, 1, 19, "Dave!:"),
+      @TK::TkSPACE .new(25, 1, 25, " "),
+      @TK::TkSYMBOL.new(26, 1, 26, ":Dave!"),
+      @TK::TkCOMMA .new(32, 1, 32, ","),
+      @TK::TkSPACE .new(33, 1, 33, " "),
+      @TK::TkSYMBOL.new(34, 1, 34, "undef:"),
+      @TK::TkSPACE .new(40, 1, 40, " "),
+      @TK::TkSYMBOL.new(41, 1, 41, ":undef"),
+      @TK::TkSPACE .new(47, 1, 47, " "),
+      @TK::TkRBRACE.new(48, 1, 48, "}"),
+      @TK::TkNL    .new(49, 1, 49, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_unary_minus
     ruby_lex = RDoc::RubyLex.new("-1", nil)
     assert_equal("-1", ruby_lex.token.value)