Merge pull request #813 from Microsoft/genericClassificationTweak

Add some heuristics in the lexical classifier to make it play better wit...

Author: CyrusNajmabadi

src/services/services.ts

@@ -4692,6 +4692,27 @@ module ts {
                 entries: []
             };
 
+            // We can run into an unfortunate interaction between the lexical and syntactic classifier
+            // when the user is typing something generic.  Consider the case where the user types:
+            //
+            //      Foo<number
+            //
+            // From the lexical classifier's perspective, 'number' is a keyword, and so the word will
+            // be classified as such.  However, from the syntactic classifier's tree-based perspective
+            // this is simply an expression with the identifier 'number' on the RHS of the less than
+            // token.  So the classification will go back to being an identifier.  The moment the user
+            // types again, number will become a keyword, then an identifier, etc. etc.
+            //
+            // To try to avoid this problem, we avoid classifying contextual keywords as keywords 
+            // when the user is potentially typing something generic.  We just can't do a good enough
+            // job at the lexical level, and so well leave it up to the syntactic classifier to make
+            // the determination.
+            //
+            // In order to determine if the user is potentially typing something generic, we use a 
+            // weak heuristic where we track < and > tokens.  It's a weak heuristic, but should
+            // work well enough in practice.
+            var angleBracketStack = 0;
+
             do {
                 token = scanner.scan();
 
@@ -4711,6 +4732,28 @@ module ts {
                         // we recognize that 'var' is actually an identifier here.
                         token = SyntaxKind.Identifier;
                     }
+                    else if (lastNonTriviaToken === SyntaxKind.Identifier &&
+                             token === SyntaxKind.LessThanToken) {
+                        // Could be the start of something generic.  Keep track of that by bumping 
+                        // up the current count of generic contexts we may be in.
+                        angleBracketStack++;
+                    }
+                    else if (token === SyntaxKind.GreaterThanToken && angleBracketStack > 0) {
+                        // If we think we're currently in something generic, then mark that that
+                        // generic entity is complete.
+                        angleBracketStack--;
+                    }
+                    else if (token === SyntaxKind.AnyKeyword ||
+                             token === SyntaxKind.StringKeyword ||
+                             token === SyntaxKind.NumberKeyword ||
+                             token === SyntaxKind.BooleanKeyword) {
+                        if (angleBracketStack > 0) {
+                            // If it looks like we're could be in something generic, don't classify this 
+                            // as a keyword.  We may just get overwritten by the syntactic classifier,
+                            // causing a noisy experience for the user.
+                            token = SyntaxKind.Identifier;
+                        }
+                    }
 
                     lastNonTriviaToken = token;
                 }

tests/cases/unittests/services/colorization.ts

@@ -231,5 +231,43 @@ describe('Colorization', function () {
                 identifier("var"),
                 finalEndOfLineState(ts.EndOfLineState.Start));
         });
+
+        it("classifies partially written generics correctly.", function () {
+            test("Foo<number",
+                ts.EndOfLineState.Start,
+                identifier("Foo"),
+                operator("<"),
+                identifier("number"),
+                finalEndOfLineState(ts.EndOfLineState.Start));
+
+            // Looks like a cast, should get classified as a keyword.
+            test("<number",
+                ts.EndOfLineState.Start,
+                operator("<"),
+                keyword("number"),
+                finalEndOfLineState(ts.EndOfLineState.Start));
+
+            // handle nesting properly.
+            test("Foo<Foo,Foo<number",
+                ts.EndOfLineState.Start,
+                identifier("Foo"),
+                operator("<"),
+                identifier("Foo"),
+                punctuation(","),
+                identifier("Foo"),
+                operator("<"),
+                identifier("number"),
+                finalEndOfLineState(ts.EndOfLineState.Start));
+
+            // no longer in something that looks generic.
+            test("Foo<Foo> number",
+                ts.EndOfLineState.Start,
+                identifier("Foo"),
+                operator("<"),
+                identifier("Foo"),
+                operator(">"
+                identifier("keyword"),
+                finalEndOfLineState(ts.EndOfLineState.Start));
+        });
     });
 });