Skip to content

[libSyntax] Record reused node IDs #18078

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jul 19, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 12 additions & 19 deletions include/swift/Parse/SyntaxParsingCache.h
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#include "swift/Syntax/SyntaxNodes.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/raw_ostream.h"
#include <unordered_set>

namespace {

Expand Down Expand Up @@ -48,10 +49,8 @@ namespace swift {
using namespace swift::syntax;

struct SyntaxReuseRegion {
/// The byte offset at which the range begins
uintptr_t Start;
/// The byte offset at which the end ends
uintptr_t End;
AbsolutePosition Start;
AbsolutePosition End;
};

class SyntaxParsingCache {
Expand All @@ -62,13 +61,8 @@ class SyntaxParsingCache {
/// the source file that is now parsed incrementally
llvm::SmallVector<SourceEdit, 4> Edits;

/// Whether or not information about reused nodes shall be recored in
/// \c ReusedRanges
bool RecordReuseInformation = false;

/// If \c RecordReuseInformation buffer offsets of ranges that have been
/// successfully looked up in this cache are stored.
std::vector<SyntaxReuseRegion> ReusedRanges;
/// The IDs of all syntax nodes that got reused are collected in this vector.
std::unordered_set<SyntaxNodeId> ReusedNodeIds;

public:
SyntaxParsingCache(SourceFileSyntax OldSyntaxTree)
Expand All @@ -86,16 +80,15 @@ class SyntaxParsingCache {
/// reused for a new syntax tree.
llvm::Optional<Syntax> lookUp(size_t NewPosition, SyntaxKind Kind);

/// Turn recording of reused ranges on
void setRecordReuseInformation() { RecordReuseInformation = true; }

/// Return the ranges of the new source file that have been successfully
/// looked up in this cache as a (start, end) pair of byte offsets in the
/// post-edit file.
std::vector<SyntaxReuseRegion> getReusedRanges() const {
return ReusedRanges;
const std::unordered_set<SyntaxNodeId> &getReusedNodeIds() const {
return ReusedNodeIds;
}

/// Get the source regions of the new source file, represented by
/// \p SyntaxTree that have been reused as part of the incremental parse.
std::vector<SyntaxReuseRegion>
getReusedRegions(const SourceFileSyntax &SyntaxTree) const;

private:
llvm::Optional<Syntax> lookUpFrom(const Syntax &Node, size_t Position,
SyntaxKind Kind);
Expand Down
20 changes: 13 additions & 7 deletions include/swift/Syntax/RawSyntax.h
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,8 @@ struct SyntaxPrintOptions {
bool PrintTrivialNodeKind = false;
};

typedef unsigned SyntaxNodeId;

/// RawSyntax - the strictly immutable, shared backing nodes for all syntax.
///
/// This is implementation detail - do not expose it in public API.
Expand All @@ -223,10 +225,10 @@ class RawSyntax final

/// The ID that shall be used for the next node that is created and does not
/// have a manually specified id
static unsigned NextFreeNodeId;
static SyntaxNodeId NextFreeNodeId;

/// An ID of this node that is stable across incremental parses
unsigned NodeId;
SyntaxNodeId NodeId;

union {
uint64_t OpaqueBits;
Expand Down Expand Up @@ -283,13 +285,13 @@ class RawSyntax final
/// the caller needs to assure that the node ID has not been used yet.
RawSyntax(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout,
SourcePresence Presence, bool ManualMemory,
llvm::Optional<unsigned> NodeId);
llvm::Optional<SyntaxNodeId> NodeId);
/// Constructor for creating token nodes
/// If \p NodeId is \c None, the next free NodeId is used, if it is passed,
/// the caller needs to assure that the NodeId has not been used yet.
RawSyntax(tok TokKind, OwnedString Text, ArrayRef<TriviaPiece> LeadingTrivia,
ArrayRef<TriviaPiece> TrailingTrivia, SourcePresence Presence,
bool ManualMemory, llvm::Optional<unsigned> NodeId);
bool ManualMemory, llvm::Optional<SyntaxNodeId> NodeId);

public:
~RawSyntax();
Expand All @@ -312,15 +314,15 @@ class RawSyntax final
static RC<RawSyntax> make(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout,
SourcePresence Presence,
SyntaxArena *Arena = nullptr,
llvm::Optional<unsigned> NodeId = llvm::None);
llvm::Optional<SyntaxNodeId> NodeId = llvm::None);

/// Make a raw "token" syntax node.
static RC<RawSyntax> make(tok TokKind, OwnedString Text,
ArrayRef<TriviaPiece> LeadingTrivia,
ArrayRef<TriviaPiece> TrailingTrivia,
SourcePresence Presence,
SyntaxArena *Arena = nullptr,
llvm::Optional<unsigned> NodeId = llvm::None);
llvm::Optional<SyntaxNodeId> NodeId = llvm::None);

/// Make a missing raw "layout" syntax node.
static RC<RawSyntax> missing(SyntaxKind Kind, SyntaxArena *Arena = nullptr) {
Expand Down Expand Up @@ -349,7 +351,7 @@ class RawSyntax final
}

/// Get an ID for this node that is stable across incremental parses
unsigned getId() const { return NodeId; }
SyntaxNodeId getId() const { return NodeId; }

/// Returns true if the node is "missing" in the source (i.e. it was
/// expected (or optional) but not written.
Expand Down Expand Up @@ -523,4 +525,8 @@ class RawSyntax final
} // end namespace syntax
} // end namespace swift

namespace llvm {
raw_ostream &operator<<(raw_ostream &OS, swift::syntax::AbsolutePosition Pos);
} // end namespace llvm

#endif // SWIFT_SYNTAX_RAWSYNTAX_H
2 changes: 1 addition & 1 deletion include/swift/Syntax/Syntax.h
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ class Syntax {
RC<RawSyntax> getRaw() const;

/// Get an ID for this node that is stable across incremental parses
unsigned getId() const { return getRaw()->getId(); }
SyntaxNodeId getId() const { return getRaw()->getId(); }

/// Get the number of child nodes in this piece of syntax, not including
/// tokens.
Expand Down
4 changes: 2 additions & 2 deletions include/swift/Syntax/SyntaxClassifier.h.gyb
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ class SyntaxClassifier: public SyntaxVisitor {
ForceClassification(ForceClassification) {}
};

std::map<unsigned, SyntaxClassification> ClassifiedTokens;
std::map<SyntaxNodeId, SyntaxClassification> ClassifiedTokens;
/// The top classification of this stack determines the color of identifiers
std::stack<ContextStackEntry, llvm::SmallVector<ContextStackEntry, 16>> ContextStack;

Expand Down Expand Up @@ -101,7 +101,7 @@ class SyntaxClassifier: public SyntaxVisitor {
% end

public:
std::map<unsigned, SyntaxClassification> classify(Syntax Node) {
std::map<SyntaxNodeId, SyntaxClassification> classify(Syntax Node) {
// Clean up the environment
ContextStack = std::stack<ContextStackEntry, llvm::SmallVector<ContextStackEntry, 16>>();
ContextStack.push({SyntaxClassification::None, false});
Expand Down
53 changes: 49 additions & 4 deletions lib/Parse/SyntaxParsingCache.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
//===----------------------------------------------------------------------===//

#include "swift/Parse/SyntaxParsingCache.h"
#include "swift/Syntax/SyntaxVisitor.h"

using namespace swift;
using namespace swift::syntax;
Expand Down Expand Up @@ -84,10 +85,54 @@ llvm::Optional<Syntax> SyntaxParsingCache::lookUp(size_t NewPosition,

auto Node = lookUpFrom(OldSyntaxTree, OldPosition, Kind);
if (Node.hasValue()) {
if (RecordReuseInformation) {
ReusedRanges.push_back(
{NewPosition, NewPosition + Node->getTextLength()});
}
ReusedNodeIds.insert(Node->getId());
}
return Node;
}

std::vector<SyntaxReuseRegion>
SyntaxParsingCache::getReusedRegions(const SourceFileSyntax &SyntaxTree) const {
/// Determines the reused source regions from reused syntax node IDs
class ReusedRegionsCollector : public SyntaxVisitor {
std::unordered_set<SyntaxNodeId> ReusedNodeIds;
std::vector<SyntaxReuseRegion> ReusedRegions;

bool didReuseNode(SyntaxNodeId NodeId) {
return ReusedNodeIds.count(NodeId) > 0;
}

public:
ReusedRegionsCollector(std::unordered_set<SyntaxNodeId> ReusedNodeIds)
: ReusedNodeIds(ReusedNodeIds) {}

const std::vector<SyntaxReuseRegion> &getReusedRegions() {
std::sort(ReusedRegions.begin(), ReusedRegions.end(),
[](const SyntaxReuseRegion &Lhs,
const SyntaxReuseRegion &Rhs) -> bool {
return Lhs.Start.getOffset() < Rhs.Start.getOffset();
});
return ReusedRegions;
}

void visit(Syntax Node) override {
if (didReuseNode(Node.getId())) {
// Node has been reused, add it to the list
auto Start = Node.getAbsolutePositionBeforeLeadingTrivia();
auto End = Node.getAbsoluteEndPositionAfterTrailingTrivia();
ReusedRegions.push_back({Start, End});
} else {
SyntaxVisitor::visit(Node);
}
}

void collectReusedRegions(SourceFileSyntax Node) {
assert(ReusedRegions.empty() &&
"ReusedRegionsCollector cannot be reused");
Node.accept(*this);
}
};

ReusedRegionsCollector ReuseRegionsCollector(getReusedNodeIds());
ReuseRegionsCollector.collectReusedRegions(SyntaxTree);
return ReuseRegionsCollector.getReusedRegions();
}
5 changes: 5 additions & 0 deletions lib/Syntax/RawSyntax.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -343,3 +343,8 @@ void RawSyntax::Profile(llvm::FoldingSetNodeID &ID, tok TokKind,
for (auto &Piece : TrailingTrivia)
Piece.Profile(ID);
}

llvm::raw_ostream &llvm::operator<<(raw_ostream &OS, AbsolutePosition Pos) {
Pos.printLineAndColumn(OS);
return OS;
}
41 changes: 13 additions & 28 deletions tools/SourceKit/lib/SwiftLang/SwiftEditor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -562,12 +562,12 @@ struct SwiftSyntaxMap {
class SyntaxToSyntaxMapConverter : public SyntaxVisitor {
SwiftSyntaxMap &SyntaxMap;

std::map<unsigned, SyntaxClassification> TokenClassifications;
std::map<SyntaxNodeId, SyntaxClassification> TokenClassifications;

public:
SyntaxToSyntaxMapConverter(
SwiftSyntaxMap &SyntaxMap,
std::map<unsigned, SyntaxClassification> TokenClassifications)
std::map<SyntaxNodeId, SyntaxClassification> TokenClassifications)
: SyntaxMap(SyntaxMap), TokenClassifications(TokenClassifications) {}

private:
Expand Down Expand Up @@ -2013,18 +2013,6 @@ SwiftEditorDocument::getSyntaxTree() const {
return Impl.SyntaxTree;
}

const SourceManager &SwiftEditorDocument::getSourceManager() const {
return Impl.SyntaxInfo->getSourceManager();
}

SourceManager &SwiftEditorDocument::getSourceManager() {
return Impl.SyntaxInfo->getSourceManager();
}

unsigned SwiftEditorDocument::getBufferID() const {
return Impl.SyntaxInfo->getBufferID();
}

std::string SwiftEditorDocument::getFilePath() const { return Impl.FilePath; }

bool SwiftEditorDocument::hasUpToDateAST() const {
Expand Down Expand Up @@ -2371,7 +2359,6 @@ void SwiftLangSupport::editorReplaceText(StringRef Name,
if (EditorDoc->getSyntaxTree().hasValue()) {
SyntaxCache.emplace(EditorDoc->getSyntaxTree().getValue());
SyntaxCache->addEdit(Offset, Offset + Length, Buf->getBufferSize());
SyntaxCache->setRecordReuseInformation();
}

SyntaxParsingCache *SyntaxCachePtr = nullptr;
Expand All @@ -2386,37 +2373,35 @@ void SwiftLangSupport::editorReplaceText(StringRef Name,
// Avoid computing the reused ranges if the consumer doesn't care about
// them
if (Consumer.syntaxReuseInfoEnabled()) {
auto ReuseRegions = SyntaxCache->getReusedRanges();
auto &SyntaxTree = EditorDoc->getSyntaxTree();
auto ReuseRegions = SyntaxCache->getReusedRegions(*SyntaxTree);

// Abstract away from SyntaxReuseRegions to std::pair<unsigned, unsigned>
// so that SourceKit doesn't have to import swiftParse
std::vector<SourceFileRange> ReuseRegionOffsets;
ReuseRegionOffsets.reserve(ReuseRegions.size());
for (auto ReuseRegion : ReuseRegions) {
auto Start = ReuseRegion.Start;
auto End = ReuseRegion.End;
auto Start = ReuseRegion.Start.getOffset();
auto End = ReuseRegion.End.getOffset();
ReuseRegionOffsets.push_back({Start, End});
}
Consumer.handleSyntaxReuseRegions(ReuseRegionOffsets);
}
if (LogReuseRegions) {
auto &SyntaxTree = EditorDoc->getSyntaxTree();
auto ReuseRegions = SyntaxCache->getReusedRegions(*SyntaxTree);
LOG_SECTION("SyntaxCache", InfoHighPrio) {
Log->getOS() << "Reused ";

bool FirstIteration = true;
unsigned LastPrintedBufferID;
for (auto ReuseRegion : SyntaxCache->getReusedRanges()) {
for (auto ReuseRegion : ReuseRegions) {
if (!FirstIteration) {
Log->getOS() << ", ";
} else {
FirstIteration = false;
}

const SourceManager &SM = EditorDoc->getSourceManager();
unsigned BufferID = EditorDoc->getBufferID();
auto Start = SM.getLocForOffset(BufferID, ReuseRegion.Start);
auto End = SM.getLocForOffset(BufferID, ReuseRegion.End);

Start.print(Log->getOS(), SM, LastPrintedBufferID);
Log->getOS() << " - ";
End.print(Log->getOS(), SM, LastPrintedBufferID);
Log->getOS() << ReuseRegion.Start << " - " << ReuseRegion.End;
}
}
}
Expand Down
6 changes: 0 additions & 6 deletions tools/SourceKit/lib/SwiftLang/SwiftLangSupport.h
Original file line number Diff line number Diff line change
Expand Up @@ -110,12 +110,6 @@ class SwiftEditorDocument :

const llvm::Optional<swift::SourceFileSyntax> &getSyntaxTree() const;

const swift::SourceManager &getSourceManager() const;
swift::SourceManager &getSourceManager();

/// Get the buffer ID of this file in its source manager
unsigned getBufferID() const;

std::string getFilePath() const;

/// Whether or not the AST stored for this document is up-to-date or just an
Expand Down
4 changes: 2 additions & 2 deletions tools/swift-ide-test/swift-ide-test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -960,7 +960,7 @@ class PrintSyntaxColorWalker : public ide::SyntaxModelWalker {
class ColoredSyntaxTreePrinter : public SyntaxVisitor {
llvm::raw_ostream &OS;

std::map<unsigned, SyntaxClassification> TokenClassifications;
std::map<SyntaxNodeId, SyntaxClassification> TokenClassifications;

/// The name of the tag that is currently open
StringRef CurrentTag;
Expand All @@ -972,7 +972,7 @@ class ColoredSyntaxTreePrinter : public SyntaxVisitor {
public:
ColoredSyntaxTreePrinter(
llvm::raw_ostream &OS,
std::map<unsigned, SyntaxClassification> TokenClassifications)
std::map<SyntaxNodeId, SyntaxClassification> TokenClassifications)
: OS(OS), TokenClassifications(TokenClassifications) {}

private:
Expand Down
Loading