[clang][deps] Serialize JSON without creating intermediate objects

[jansvoboda11]

The dependency scanner uses the llvm::json library for outputting the dependency information. Until now, it created an in-memory representation of the dependency graph using the llvm::json::Object hierarchy. This not only creates unnecessary copies of the data, but also forces lexicographical ordering of attributes in the output, both of which I'd like to avoid. This patch adopts the llvm::json::OStream API instead and reorders the attribute printing logic such that the existing lexicographical ordering is preserved (for now).

[llvmbot]

@llvm/pr-subscribers-clang

Author: Jan Svoboda (jansvoboda11)

Changes

The dependency scanner uses the llvm::json library for outputting the dependency information. Until now, it created an in-memory representation of the dependency graph using the llvm::json::Object hierarchy. This not only creates unnecessary copies of the data, but also forces lexicographical ordering of attributes in the output, both of which I'd like to avoid. This patch adopts the llvm::json::OStream API instead and reorders the attribute printing logic such that the existing lexicographical ordering is preserved (for now).

---

Full diff: https://github.com/llvm/llvm-project/pull/111734.diff

1 Files Affected:

• (modified) clang/tools/clang-scan-deps/ClangScanDeps.cpp (+87-72)

diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
index b642a37c79e980..0c1774e60e7cc2 100644
--- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -330,38 +330,46 @@ handleMakeDependencyToolResult(const std::string &Input,
   return false;
 }
 
-static llvm::json::Array toJSONSorted(const llvm::StringSet<> &Set) {
-  std::vector<llvm::StringRef> Strings;
-  for (auto &&I : Set)
-    Strings.push_back(I.getKey());
+template <typename Container>
+static auto toJSONStrings(llvm::json::OStream &JOS, Container &&Strings) {
+  return [&JOS, Strings = std::forward<Container>(Strings)]() {
+    for (StringRef Str : Strings)
+      JOS.value(Str);
+  };
+}
+
+static auto toJSONSorted(llvm::json::OStream &JOS,
+                         const llvm::StringSet<> &Set) {
+  SmallVector<StringRef> Strings(Set.keys());
   llvm::sort(Strings);
-  return llvm::json::Array(Strings);
+  return toJSONStrings(JOS, std::move(Strings));
 }
 
 // Technically, we don't need to sort the dependency list to get determinism.
 // Leaving these be will simply preserve the import order.
-static llvm::json::Array toJSONSorted(std::vector<ModuleID> V) {
+static auto toJSONSorted(llvm::json::OStream &JOS, std::vector<ModuleID> V) {
   llvm::sort(V);
-
-  llvm::json::Array Ret;
-  for (const ModuleID &MID : V)
-    Ret.push_back(llvm::json::Object(
-        {{"module-name", MID.ModuleName}, {"context-hash", MID.ContextHash}}));
-  return Ret;
+  return [&JOS, V = std::move(V)]() {
+    for (const ModuleID &MID : V)
+      JOS.object([&]() {
+        JOS.attribute("context-hash", StringRef(MID.ContextHash));
+        JOS.attribute("module-name", StringRef(MID.ModuleName));
+      });
+  };
 }
 
-static llvm::json::Array
-toJSONSorted(llvm::SmallVector<Module::LinkLibrary, 2> &LinkLibs) {
-  llvm::sort(LinkLibs, [](const Module::LinkLibrary &lhs,
-                          const Module::LinkLibrary &rhs) {
-    return lhs.Library < rhs.Library;
+static auto toJSONSorted(llvm::json::OStream &JOS,
+                         SmallVector<Module::LinkLibrary, 2> LinkLibs) {
+  llvm::sort(LinkLibs, [](const auto &LHS, const auto &RHS) {
+    return LHS.Library < RHS.Library;
   });
-
-  llvm::json::Array Ret;
-  for (const Module::LinkLibrary &LL : LinkLibs)
-    Ret.push_back(llvm::json::Object(
-        {{"link-name", LL.Library}, {"isFramework", LL.IsFramework}}));
-  return Ret;
+  return [&JOS, LinkLibs = std::move(LinkLibs)]() {
+    for (const auto &LL : LinkLibs)
+      JOS.object([&]() {
+        JOS.attribute("isFramework", LL.IsFramework);
+        JOS.attribute("link-name", StringRef(LL.Library));
+      });
+  };
 }
 
 // Thread safe.
@@ -450,58 +458,65 @@ class FullDeps {
       ModuleIDs.push_back(M.first);
     llvm::sort(ModuleIDs);
 
-    using namespace llvm::json;
-
-    Array OutModules;
-    for (auto &&ModID : ModuleIDs) {
-      auto &MD = Modules[ModID];
-      Object O{{"name", MD.ID.ModuleName},
-               {"context-hash", MD.ID.ContextHash},
-               {"file-deps", toJSONSorted(MD.FileDeps)},
-               {"clang-module-deps", toJSONSorted(MD.ClangModuleDeps)},
-               {"clang-modulemap-file", MD.ClangModuleMapFile},
-               {"command-line", MD.getBuildArguments()},
-               {"link-libraries", toJSONSorted(MD.LinkLibraries)}};
-      OutModules.push_back(std::move(O));
-    }
-
-    Array TUs;
-    for (auto &&I : Inputs) {
-      Array Commands;
-      if (I.DriverCommandLine.empty()) {
-        for (const auto &Cmd : I.Commands) {
-          Object O{
-              {"input-file", I.FileName},
-              {"clang-context-hash", I.ContextHash},
-              {"file-deps", I.FileDeps},
-              {"clang-module-deps", toJSONSorted(I.ModuleDeps)},
-              {"executable", Cmd.Executable},
-              {"command-line", Cmd.Arguments},
-          };
-          Commands.push_back(std::move(O));
+    llvm::json::OStream JOS(OS, /*IndentSize=*/2);
+
+    JOS.object([&]() {
+      JOS.attributeArray("modules", [&]() {
+        for (auto &&ModID : ModuleIDs) {
+          auto &MD = Modules[ModID];
+          JOS.object([&]() {
+            JOS.attributeArray("clang-module-deps",
+                               toJSONSorted(JOS, MD.ClangModuleDeps));
+            JOS.attribute("clang-modulemap-file",
+                          StringRef(MD.ClangModuleMapFile));
+            JOS.attributeArray("command-line",
+                               toJSONStrings(JOS, MD.getBuildArguments()));
+            JOS.attribute("context-hash", StringRef(MD.ID.ContextHash));
+            JOS.attributeArray("file-deps", toJSONSorted(JOS, MD.FileDeps));
+            JOS.attributeArray("link-libraries",
+                               toJSONSorted(JOS, MD.LinkLibraries));
+            JOS.attribute("name", StringRef(MD.ID.ModuleName));
+          });
         }
-      } else {
-        Object O{
-            {"input-file", I.FileName},
-            {"clang-context-hash", I.ContextHash},
-            {"file-deps", I.FileDeps},
-            {"clang-module-deps", toJSONSorted(I.ModuleDeps)},
-            {"executable", "clang"},
-            {"command-line", I.DriverCommandLine},
-        };
-        Commands.push_back(std::move(O));
-      }
-      TUs.push_back(Object{
-          {"commands", std::move(Commands)},
       });
-    }
-
-    Object Output{
-        {"modules", std::move(OutModules)},
-        {"translation-units", std::move(TUs)},
-    };
 
-    OS << llvm::formatv("{0:2}\n", Value(std::move(Output)));
+      JOS.attributeArray("translation-units", [&]() {
+        for (auto &&I : Inputs) {
+          JOS.object([&]() {
+            JOS.attributeArray("commands", [&]() {
+              if (I.DriverCommandLine.empty()) {
+                for (const auto &Cmd : I.Commands) {
+                  JOS.object([&]() {
+                    JOS.attribute("clang-context-hash",
+                                  StringRef(I.ContextHash));
+                    JOS.attributeArray("clang-module-deps",
+                                       toJSONSorted(JOS, I.ModuleDeps));
+                    JOS.attributeArray("command-line",
+                                       toJSONStrings(JOS, Cmd.Arguments));
+                    JOS.attribute("executable", StringRef(Cmd.Executable));
+                    JOS.attributeArray("file-deps",
+                                       toJSONStrings(JOS, I.FileDeps));
+                    JOS.attribute("input-file", StringRef(I.FileName));
+                  });
+                }
+              } else {
+                JOS.object([&]() {
+                  JOS.attribute("clang-context-hash", StringRef(I.ContextHash));
+                  JOS.attributeArray("clang-module-deps",
+                                     toJSONSorted(JOS, I.ModuleDeps));
+                  JOS.attributeArray("command-line",
+                                     toJSONStrings(JOS, I.DriverCommandLine));
+                  JOS.attribute("executable", "clang");
+                  JOS.attributeArray("file-deps",
+                                     toJSONStrings(JOS, I.FileDeps));
+                  JOS.attribute("input-file", StringRef(I.FileName));
+                });
+              }
+            });
+          });
+        }
+      });
+    });
   }
 
 private: