Skip to content

[Syntax] test diagnostics in Lexer with libSyntax #14954

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 3, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion include/swift/Parse/Parser.h
Original file line number Diff line number Diff line change
Expand Up @@ -1444,7 +1444,8 @@ tokenizeWithTrivia(const LangOptions &LangOpts,
const SourceManager &SM,
unsigned BufferID,
unsigned Offset = 0,
unsigned EndOffset = 0);
unsigned EndOffset = 0,
DiagnosticEngine *Diags = nullptr);
} // end namespace swift

#endif
1 change: 1 addition & 0 deletions include/swift/Subsystems.h
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ namespace swift {
std::vector<Token> tokenize(const LangOptions &LangOpts,
const SourceManager &SM, unsigned BufferID,
unsigned Offset = 0, unsigned EndOffset = 0,
DiagnosticEngine *Diags = nullptr,
bool KeepComments = true,
bool TokenizeInterpolatedString = true,
ArrayRef<Token> SplitTokens = ArrayRef<Token>());
Expand Down
10 changes: 8 additions & 2 deletions lib/Parse/Parser.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ namespace swift {
template <typename DF>
void tokenize(const LangOptions &LangOpts, const SourceManager &SM,
unsigned BufferID, unsigned Offset, unsigned EndOffset,
DiagnosticEngine * Diags,
CommentRetentionMode RetainComments,
TriviaRetentionMode TriviaRetention,
bool TokenizeInterpolatedString, ArrayRef<Token> SplitTokens,
Expand All @@ -56,7 +57,7 @@ void tokenize(const LangOptions &LangOpts, const SourceManager &SM,
if (Offset == 0 && EndOffset == 0)
EndOffset = SM.getRangeForBuffer(BufferID).getByteLength();

Lexer L(LangOpts, SM, BufferID, /*Diags=*/nullptr, /*InSILMode=*/false,
Lexer L(LangOpts, SM, BufferID, Diags, /*InSILMode=*/false,
RetainComments, TriviaRetention, Offset, EndOffset);

auto TokComp = [&](const Token &A, const Token &B) {
Expand Down Expand Up @@ -260,6 +261,7 @@ static void getStringPartTokens(const Token &Tok, const LangOptions &LangOpts,

std::vector<Token> NewTokens = swift::tokenize(LangOpts, SM, BufID,
Offset, EndOffset,
/*Diags=*/nullptr,
/*KeepComments=*/true);
Toks.insert(Toks.end(), NewTokens.begin(), NewTokens.end());

Expand All @@ -278,12 +280,14 @@ static void getStringPartTokens(const Token &Tok, const LangOptions &LangOpts,
std::vector<Token> swift::tokenize(const LangOptions &LangOpts,
const SourceManager &SM, unsigned BufferID,
unsigned Offset, unsigned EndOffset,
DiagnosticEngine *Diags,
bool KeepComments,
bool TokenizeInterpolatedString,
ArrayRef<Token> SplitTokens) {
std::vector<Token> Tokens;

tokenize(LangOpts, SM, BufferID, Offset, EndOffset,
Diags,
KeepComments ? CommentRetentionMode::ReturnAsTokens
: CommentRetentionMode::AttachToNextToken,
TriviaRetentionMode::WithoutTrivia, TokenizeInterpolatedString,
Expand All @@ -299,13 +303,15 @@ std::vector<Token> swift::tokenize(const LangOptions &LangOpts,
std::vector<std::pair<RC<syntax::RawSyntax>, syntax::AbsolutePosition>>
swift::tokenizeWithTrivia(const LangOptions &LangOpts, const SourceManager &SM,
unsigned BufferID, unsigned Offset,
unsigned EndOffset) {
unsigned EndOffset,
DiagnosticEngine *Diags) {
std::vector<std::pair<RC<syntax::RawSyntax>, syntax::AbsolutePosition>>
Tokens;
syntax::AbsolutePosition RunningPos;

tokenize(
LangOpts, SM, BufferID, Offset, EndOffset,
Diags,
CommentRetentionMode::AttachToNextToken, TriviaRetentionMode::WithTrivia,
/*TokenizeInterpolatedString=*/false,
/*SplitTokens=*/ArrayRef<Token>(),
Expand Down
5 changes: 5 additions & 0 deletions test/Syntax/lexer_invalid_nul.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
// RUN: cat %s | tr '\132' '\0' > %t
// RUN: %swift-syntax-test -input-source-filename %t -dump-full-tokens 2>&1 >/dev/null | %FileCheck %t

// CHECK: 5:18: warning: nul character embedded in middle of file
let a = 3 // nul(Z)
12 changes: 7 additions & 5 deletions tools/swift-syntax-test/swift-syntax-test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -112,11 +112,13 @@ namespace {
int getTokensFromFile(unsigned BufferID,
LangOptions &LangOpts,
SourceManager &SourceMgr,
DiagnosticEngine &Diags,
swift::DiagnosticEngine &Diags,
std::vector<std::pair<RC<syntax::RawSyntax>,
syntax::AbsolutePosition>> &Tokens) {
Tokens = tokenizeWithTrivia(LangOpts, SourceMgr, BufferID);
return Diags.hadAnyError() ? EXIT_FAILURE : EXIT_SUCCESS;
Tokens = tokenizeWithTrivia(LangOpts, SourceMgr, BufferID,
/*Offset=*/0, /*EndOffset=*/0,
&Diags);
return EXIT_SUCCESS;
}


Expand Down Expand Up @@ -191,7 +193,7 @@ int doFullLexRoundTrip(const StringRef InputFilename) {
TokAndPos.first->print(llvm::outs(), {});
}

return Diags.hadAnyError() ? EXIT_FAILURE : EXIT_SUCCESS;
return EXIT_SUCCESS;
}

int doDumpRawTokenSyntax(const StringRef InputFilename) {
Expand All @@ -215,7 +217,7 @@ int doDumpRawTokenSyntax(const StringRef InputFilename) {
llvm::outs() << "\n";
}

return Diags.hadAnyError() ? EXIT_FAILURE : EXIT_SUCCESS;
return EXIT_SUCCESS;
}

int doFullParseRoundTrip(const char *MainExecutablePath,
Expand Down
4 changes: 2 additions & 2 deletions unittests/Parse/LexerTests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ class LexerTest : public ::testing::Test {
if (KeepEOF)
Toks = tokenizeAndKeepEOF(BufID);
else
Toks = tokenize(LangOpts, SourceMgr, BufID, 0, 0, KeepComments);
Toks = tokenize(LangOpts, SourceMgr, BufID, 0, 0, /*Diags=*/nullptr, KeepComments);
EXPECT_EQ(ExpectedTokens.size(), Toks.size());
for (unsigned i = 0, e = ExpectedTokens.size(); i != e; ++i) {
EXPECT_EQ(ExpectedTokens[i], Toks[i].getKind()) << "i = " << i;
Expand Down Expand Up @@ -685,7 +685,7 @@ TEST_F(LexerTest, TokenizePlaceholder) {
TEST_F(LexerTest, NoPlaceholder) {
auto checkTok = [&](StringRef Source) {
unsigned BufID = SourceMgr.addMemBufferCopy(Source);
std::vector<Token> Toks = tokenize(LangOpts, SourceMgr, BufID, 0, 0, false);
std::vector<Token> Toks = tokenize(LangOpts, SourceMgr, BufID, 0, 0, /*Diags=*/nullptr, false);
ASSERT_FALSE(Toks.empty());
EXPECT_NE(tok::identifier, Toks[0].getKind());
};
Expand Down
1 change: 1 addition & 0 deletions unittests/Parse/TokenizerTests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ class TokenizerTest : public ::testing::Test {
BufID,
/* Offset = */ 0,
/* EndOffset = */ 0,
/* Diags = */nullptr,
/* KeepComments = */ true,
/* TokenizeInterpolatedString = */ true,
SplitTokens);
Expand Down