Skip to content

Commit d627a27

Browse files
committed
[AST] Add generator for source location introspection
Generate a json file containing descriptions of AST classes and their public accessors which return SourceLocation or SourceRange. Use the JSON file to generate a C++ API and implementation for accessing the source locations and method names for accessing them for a given AST node. This new API can be used to implement 'srcloc' output in clang-query: http://ce.steveire.com/z/m_kTIo In this first version of this feature, only the accessors for Stmt classes are generated, not Decls, TypeLocs etc. Those can be added after this change is reviewed, as this change is mostly about infrastructure of these code generators. Differential Revision: https://reviews.llvm.org/D93164
1 parent 08d33aa commit d627a27

File tree

12 files changed

+962
-0
lines changed

12 files changed

+962
-0
lines changed
Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
//===- NodeIntrospection.h ------------------------------------*- C++ -*---===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
//
9+
// This file contains the implementation of the NodeIntrospection.
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
#ifndef LLVM_CLANG_TOOLING_NODEINTROSPECTION_H
14+
#define LLVM_CLANG_TOOLING_NODEINTROSPECTION_H
15+
16+
#include "clang/AST/ASTTypeTraits.h"
17+
#include "clang/AST/DeclarationName.h"
18+
19+
#include <memory>
20+
#include <set>
21+
22+
namespace clang {
23+
24+
class Stmt;
25+
26+
namespace tooling {
27+
28+
class LocationCall {
29+
public:
30+
enum LocationCallFlags { NoFlags, ReturnsPointer, IsCast };
31+
LocationCall(std::shared_ptr<LocationCall> on, std::string name,
32+
LocationCallFlags flags = NoFlags)
33+
: m_on(on), m_name(name), m_flags(flags) {}
34+
LocationCall(std::shared_ptr<LocationCall> on, std::string name,
35+
std::vector<std::string> const &args,
36+
LocationCallFlags flags = NoFlags)
37+
: m_on(on), m_name(name), m_flags(flags) {}
38+
39+
LocationCall *on() const { return m_on.get(); }
40+
StringRef name() const { return m_name; }
41+
std::vector<std::string> const &args() const { return m_args; }
42+
bool returnsPointer() const { return m_flags & ReturnsPointer; }
43+
bool isCast() const { return m_flags & IsCast; }
44+
45+
private:
46+
std::shared_ptr<LocationCall> m_on;
47+
std::string m_name;
48+
std::vector<std::string> m_args;
49+
LocationCallFlags m_flags;
50+
};
51+
52+
class LocationCallFormatterCpp {
53+
public:
54+
static std::string format(LocationCall *Call);
55+
};
56+
57+
namespace internal {
58+
struct RangeLessThan {
59+
bool operator()(
60+
std::pair<SourceRange, std::shared_ptr<LocationCall>> const &LHS,
61+
std::pair<SourceRange, std::shared_ptr<LocationCall>> const &RHS) const;
62+
};
63+
} // namespace internal
64+
65+
template <typename T, typename U, typename Comp = std::less<std::pair<T, U>>>
66+
using UniqueMultiMap = std::set<std::pair<T, U>, Comp>;
67+
68+
using SourceLocationMap =
69+
UniqueMultiMap<SourceLocation, std::shared_ptr<LocationCall>>;
70+
using SourceRangeMap =
71+
UniqueMultiMap<SourceRange, std::shared_ptr<LocationCall>,
72+
internal::RangeLessThan>;
73+
74+
struct NodeLocationAccessors {
75+
SourceLocationMap LocationAccessors;
76+
SourceRangeMap RangeAccessors;
77+
};
78+
79+
namespace NodeIntrospection {
80+
NodeLocationAccessors GetLocations(clang::Stmt const *Object);
81+
NodeLocationAccessors GetLocations(clang::DynTypedNode const &Node);
82+
} // namespace NodeIntrospection
83+
} // namespace tooling
84+
} // namespace clang
85+
#endif

clang/lib/Tooling/CMakeLists.txt

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,78 @@ add_subdirectory(Core)
88
add_subdirectory(Inclusions)
99
add_subdirectory(Refactoring)
1010
add_subdirectory(ASTDiff)
11+
add_subdirectory(DumpTool)
1112
add_subdirectory(Syntax)
1213
add_subdirectory(DependencyScanning)
1314
add_subdirectory(Transformer)
1415

16+
find_package(Python3 COMPONENTS Interpreter)
17+
18+
# The generation of ASTNodeAPI.json takes a long time in a
19+
# Debug build due to parsing AST.h. Disable the processing
20+
# but setting CLANG_TOOLING_BUILD_AST_INTROSPECTION as an
21+
# internal hidden setting to override.
22+
# When the processing is disabled, a trivial/empty JSON
23+
# file is generated by clang-ast-dump and generate_cxx_src_locs.py
24+
# generates the same API, but with a trivial implementation.
25+
26+
option(CLANG_TOOLING_BUILD_AST_INTROSPECTION "Enable AST introspection" TRUE)
27+
set(skip_expensive_processing $<OR:$<CONFIG:Debug>,$<NOT:$<BOOL:${CLANG_TOOLING_BUILD_AST_INTROSPECTION}>>>)
28+
29+
add_custom_command(
30+
COMMENT Generate ASTNodeAPI.json
31+
OUTPUT ${CMAKE_BINARY_DIR}/ASTNodeAPI.json
32+
DEPENDS clang-ast-dump clang-headers
33+
COMMAND
34+
$<TARGET_FILE:clang-ast-dump>
35+
# Skip this in debug mode because parsing AST.h is too slow
36+
--skip-processing=${skip_expensive_processing}
37+
--astheader=${CMAKE_SOURCE_DIR}/../clang/include/clang/AST/AST.h
38+
-I ${CMAKE_BINARY_DIR}/lib/clang/${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}.${LLVM_VERSION_PATCH}/include
39+
-I ${CMAKE_SOURCE_DIR}/../clang/include
40+
-I ${CMAKE_BINARY_DIR}/tools/clang/include/
41+
-I ${CMAKE_BINARY_DIR}/include
42+
-I ${CMAKE_SOURCE_DIR}/include
43+
--json-output-path ${CMAKE_BINARY_DIR}/ASTNodeAPI.json
44+
)
45+
46+
add_custom_target(run-ast-api-dump-tool
47+
DEPENDS ${CMAKE_BINARY_DIR}/ASTNodeAPI.json
48+
)
49+
50+
# Replace the last lib component of the current binary directory with include
51+
string(FIND ${CMAKE_CURRENT_BINARY_DIR} "/lib/" PATH_LIB_START REVERSE)
52+
if(PATH_LIB_START EQUAL -1)
53+
message(FATAL_ERROR "Couldn't find lib component in binary directory")
54+
endif()
55+
math(EXPR PATH_LIB_END "${PATH_LIB_START}+5")
56+
string(SUBSTRING ${CMAKE_CURRENT_BINARY_DIR} 0 ${PATH_LIB_START} PATH_HEAD)
57+
string(SUBSTRING ${CMAKE_CURRENT_BINARY_DIR} ${PATH_LIB_END} -1 PATH_TAIL)
58+
string(CONCAT BINARY_INCLUDE_DIR ${PATH_HEAD} "/include/clang/" ${PATH_TAIL})
59+
60+
add_custom_command(
61+
COMMENT Generate NodeIntrospection.inc
62+
OUTPUT ${BINARY_INCLUDE_DIR}/NodeIntrospection.inc
63+
DEPENDS ${CMAKE_BINARY_DIR}/ASTNodeAPI.json ${CMAKE_CURRENT_SOURCE_DIR}/DumpTool/generate_cxx_src_locs.py
64+
COMMAND
65+
${CMAKE_COMMAND} -E make_directory
66+
${CMAKE_CURRENT_BINARY_DIR}/generated/
67+
COMMAND
68+
${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/DumpTool/generate_cxx_src_locs.py
69+
--json-input-path ${CMAKE_BINARY_DIR}/ASTNodeAPI.json
70+
--output-file generated/NodeIntrospection.inc
71+
--empty-implementation ${skip_expensive_processing}
72+
COMMAND
73+
${CMAKE_COMMAND} -E copy_if_different
74+
${CMAKE_CURRENT_BINARY_DIR}/generated/NodeIntrospection.inc
75+
${BINARY_INCLUDE_DIR}/NodeIntrospection.inc
76+
)
77+
78+
add_custom_target(run-ast-api-generate-tool
79+
DEPENDS
80+
${BINARY_INCLUDE_DIR}/NodeIntrospection.inc
81+
)
82+
1583
add_clang_library(clangTooling
1684
AllTUsExecution.cpp
1785
ArgumentsAdjusters.cpp
@@ -27,6 +95,8 @@ add_clang_library(clangTooling
2795
Refactoring.cpp
2896
RefactoringCallbacks.cpp
2997
StandaloneExecution.cpp
98+
NodeIntrospection.cpp
99+
${BINARY_INCLUDE_DIR}/NodeIntrospection.inc
30100
Tooling.cpp
31101

32102
DEPENDS

clang/lib/Tooling/DumpTool/APIData.h

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
//===- APIData.h ---------------------------------------------*- C++ -*----===//
2+
//
3+
// The LLVM Compiler Infrastructure
4+
//
5+
// This file is distributed under the University of Illinois Open Source
6+
// License. See LICENSE.TXT for details.
7+
//
8+
//===----------------------------------------------------------------------===//
9+
10+
#ifndef LLVM_CLANG_LIB_TOOLING_DUMPTOOL_APIDATA_H
11+
#define LLVM_CLANG_LIB_TOOLING_DUMPTOOL_APIDATA_H
12+
13+
#include <string>
14+
#include <vector>
15+
16+
namespace clang {
17+
namespace tooling {
18+
19+
struct ClassData {
20+
21+
bool isEmpty() const {
22+
return ASTClassLocations.empty() && ASTClassRanges.empty();
23+
}
24+
25+
std::vector<std::string> ASTClassLocations;
26+
std::vector<std::string> ASTClassRanges;
27+
// TODO: Extend this with locations available via typelocs etc.
28+
};
29+
30+
} // namespace tooling
31+
} // namespace clang
32+
33+
#endif
Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
//===- ASTSrcLocProcessor.cpp --------------------------------*- C++ -*----===//
2+
//
3+
// The LLVM Compiler Infrastructure
4+
//
5+
// This file is distributed under the University of Illinois Open Source
6+
// License. See LICENSE.TXT for details.
7+
//
8+
//===----------------------------------------------------------------------===//
9+
10+
#include "ASTSrcLocProcessor.h"
11+
12+
#include "clang/Frontend/CompilerInstance.h"
13+
#include "llvm/Support/JSON.h"
14+
15+
using namespace clang::tooling;
16+
using namespace llvm;
17+
using namespace clang::ast_matchers;
18+
19+
ASTSrcLocProcessor::ASTSrcLocProcessor(StringRef JsonPath)
20+
: JsonPath(JsonPath) {
21+
22+
MatchFinder::MatchFinderOptions FinderOptions;
23+
24+
Finder = std::make_unique<MatchFinder>(std::move(FinderOptions));
25+
Finder->addMatcher(
26+
cxxRecordDecl(
27+
isDefinition(),
28+
isSameOrDerivedFrom(
29+
// TODO: Extend this with other clades
30+
namedDecl(hasName("clang::Stmt")).bind("nodeClade")),
31+
optionally(isDerivedFrom(cxxRecordDecl().bind("derivedFrom"))))
32+
.bind("className"),
33+
this);
34+
}
35+
36+
std::unique_ptr<clang::ASTConsumer>
37+
ASTSrcLocProcessor::createASTConsumer(clang::CompilerInstance &Compiler,
38+
StringRef File) {
39+
return Finder->newASTConsumer();
40+
}
41+
42+
llvm::json::Object toJSON(llvm::StringMap<std::vector<StringRef>> const &Obj) {
43+
using llvm::json::toJSON;
44+
45+
llvm::json::Object JsonObj;
46+
for (const auto &Item : Obj) {
47+
JsonObj[Item.first()] = Item.second;
48+
}
49+
return JsonObj;
50+
}
51+
52+
llvm::json::Object toJSON(llvm::StringMap<StringRef> const &Obj) {
53+
using llvm::json::toJSON;
54+
55+
llvm::json::Object JsonObj;
56+
for (const auto &Item : Obj) {
57+
JsonObj[Item.first()] = Item.second;
58+
}
59+
return JsonObj;
60+
}
61+
62+
llvm::json::Object toJSON(ClassData const &Obj) {
63+
llvm::json::Object JsonObj;
64+
65+
if (!Obj.ASTClassLocations.empty())
66+
JsonObj["sourceLocations"] = Obj.ASTClassLocations;
67+
if (!Obj.ASTClassRanges.empty())
68+
JsonObj["sourceRanges"] = Obj.ASTClassRanges;
69+
return JsonObj;
70+
}
71+
72+
llvm::json::Object toJSON(llvm::StringMap<ClassData> const &Obj) {
73+
using llvm::json::toJSON;
74+
75+
llvm::json::Object JsonObj;
76+
for (const auto &Item : Obj) {
77+
if (!Item.second.isEmpty())
78+
JsonObj[Item.first()] = ::toJSON(Item.second);
79+
}
80+
return JsonObj;
81+
}
82+
83+
void WriteJSON(std::string JsonPath,
84+
llvm::StringMap<StringRef> const &ClassInheritance,
85+
llvm::StringMap<std::vector<StringRef>> const &ClassesInClade,
86+
llvm::StringMap<ClassData> const &ClassEntries) {
87+
llvm::json::Object JsonObj;
88+
89+
using llvm::json::toJSON;
90+
91+
JsonObj["classInheritance"] = ::toJSON(ClassInheritance);
92+
JsonObj["classesInClade"] = ::toJSON(ClassesInClade);
93+
JsonObj["classEntries"] = ::toJSON(ClassEntries);
94+
95+
std::error_code EC;
96+
llvm::raw_fd_ostream JsonOut(JsonPath, EC, llvm::sys::fs::F_Text);
97+
if (EC)
98+
return;
99+
100+
llvm::json::Value JsonVal(std::move(JsonObj));
101+
JsonOut << formatv("{0:2}", JsonVal);
102+
}
103+
104+
void ASTSrcLocProcessor::generate() {
105+
WriteJSON(JsonPath, ClassInheritance, ClassesInClade, ClassEntries);
106+
}
107+
108+
std::vector<std::string>
109+
CaptureMethods(std::string TypeString, const clang::CXXRecordDecl *ASTClass,
110+
const MatchFinder::MatchResult &Result) {
111+
112+
auto publicAccessor = [](auto... InnerMatcher) {
113+
return cxxMethodDecl(isPublic(), parameterCountIs(0), isConst(),
114+
InnerMatcher...);
115+
};
116+
117+
auto BoundNodesVec =
118+
match(findAll(publicAccessor(ofClass(equalsNode(ASTClass)),
119+
returns(asString(TypeString)))
120+
.bind("classMethod")),
121+
*ASTClass, *Result.Context);
122+
123+
std::vector<std::string> Methods;
124+
for (const auto &BN : BoundNodesVec) {
125+
if (const auto *Node = BN.getNodeAs<clang::NamedDecl>("classMethod")) {
126+
// Only record the getBeginLoc etc on Stmt etc, because it will call
127+
// more-derived implementations pseudo-virtually.
128+
if ((ASTClass->getName() != "Stmt" && ASTClass->getName() != "Decl") &&
129+
(Node->getName() == "getBeginLoc" || Node->getName() == "getEndLoc" ||
130+
Node->getName() == "getSourceRange")) {
131+
continue;
132+
}
133+
// Only record the getExprLoc on Expr, because it will call
134+
// more-derived implementations pseudo-virtually.
135+
if (ASTClass->getName() != "Expr" && Node->getName() == "getExprLoc") {
136+
continue;
137+
}
138+
Methods.push_back(Node->getName().str());
139+
}
140+
}
141+
return Methods;
142+
}
143+
144+
void ASTSrcLocProcessor::run(const MatchFinder::MatchResult &Result) {
145+
146+
if (const auto *ASTClass =
147+
Result.Nodes.getNodeAs<clang::CXXRecordDecl>("className")) {
148+
149+
StringRef ClassName = ASTClass->getName();
150+
151+
ClassData CD;
152+
153+
const auto *NodeClade =
154+
Result.Nodes.getNodeAs<clang::CXXRecordDecl>("nodeClade");
155+
StringRef CladeName = NodeClade->getName();
156+
157+
if (const auto *DerivedFrom =
158+
Result.Nodes.getNodeAs<clang::CXXRecordDecl>("derivedFrom"))
159+
ClassInheritance[ClassName] = DerivedFrom->getName();
160+
161+
CD.ASTClassLocations =
162+
CaptureMethods("class clang::SourceLocation", ASTClass, Result);
163+
CD.ASTClassRanges =
164+
CaptureMethods("class clang::SourceRange", ASTClass, Result);
165+
166+
if (!CD.isEmpty()) {
167+
ClassEntries[ClassName] = CD;
168+
ClassesInClade[CladeName].push_back(ClassName);
169+
}
170+
}
171+
}

0 commit comments

Comments
 (0)