[SYCL] Use file-table-tform in SYCL offload processing in clang driver.

Clang driver's design can't handily model
(1) multiple inputs/outputs in the action graph. Because of that, for
example, sycl-post-link tool is invoked twice - once to to split the code
and produce multiple bitcode files, and secondly - to generate symbol
files for the split modules.
(2) "Clusters" of inputs/outputs, when subsets of inputs/outputs are
associated and describe different aspects of the same data. Example of
such clustering is the split module + its symbol file above. Clustering
would require support both in the driver and the tools invoked in
response to actions.

This commit moves SYCL offload processing to the "file table concept."
sycl-post-link instead of
(1) being invoked n times, once per each output type requested (once for
    device split and once for symbol file generation)
(2) outputting multiple file lists each listing outputs from the
    corresponding invocation above
is now invoked once and produces single file table output. E.g.
  [Code|Symbols|Properties]
  a_0.bc|a_0.sym|a_0.props
  a_1.bc|a_1.sym|a_1.props
This solves both problems - multiple input/output and clustering.
Combined with the file-table-tform tool, this allows for efficent handling
of multiple clusters of files (each represented as a row in the table file)
in the clang driver infrastructure.
For example, there is a real offload processing problem:
step1. sycl-post-link outputs N clusters of files
step2. "Code" file of each cluster resuilting from step1 ({a_0.bc, a_1.bc}
       in the example above) must undergo further transformations -
       translation to SPIRV and optional ahead-of-time compilation.
step3. In each cluster resulting from step1 the "Code" file needs to be
       replaced with the result of step2
step4. All the clusters are processed by the ClangOffloadWrapper tool, which
       needs to know how files are distributed into clusters and what is
       the roles of each file in a cluster - whether it is "Code", "Symbol"
       or "Properties".
To solve this, the following action graph is constructed in the clang driver:

                        column:"Code"
t1 -> [file-table-tform:extract column] -> t1a -> [for-each:] -> t1b
                                                  llvm-spirv
                                                   aot-comp
t1
   \                  column:"Code"
    [file-table-tform:replace column] -> t2 -> [ClangOffloadWrapper]
   /
t1b

where t1b is ["Code"]  and t2 is  [Code|Symbols|Properties]
              a_0.bin             a_0.bin|a_0.sym|a_0.props
              a_1.bin             a_1.bin|a_1.sym|a_1.props

Note that the graph does not change with growing number of clusters, neither
it changes when more files are added to each cluster (e.g. a "Manifest" file).

Signed-off-by: Konstantin S Bobrovsky <[email protected]>

Author: kbobrovs

clang/include/clang/Driver/Action.h

@@ -17,6 +17,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator_range.h"
+#include <initializer_list>
 #include <string>
 
 namespace llvm {
@@ -78,9 +79,10 @@ class Action {
     SYCLPostLinkJobClass,
     PartialLinkJobClass,
     BackendCompileJobClass,
+    FileTableTformJobClass,
 
     JobClassFirst = PreprocessJobClass,
-    JobClassLast = BackendCompileJobClass
+    JobClassLast = FileTableTformJobClass
   };
 
   // The offloading kind determines if this action is binded to a particular
@@ -679,6 +681,13 @@ class SYCLPostLinkJobAction : public JobAction {
   static bool classof(const Action *A) {
     return A->getKind() == SYCLPostLinkJobClass;
   }
+
+  void setRTSetsSpecConstants(bool Val) { RTSetsSpecConsts = Val; }
+
+  bool getRTSetsSpecConstants() const { return RTSetsSpecConsts; }
+
+private:
+  bool RTSetsSpecConsts = true;
 };
 
 class PartialLinkJobAction : public JobAction {
@@ -705,6 +714,47 @@ class BackendCompileJobAction : public JobAction {
   }
 };
 
+// Represents a file table transformation action. The order of inputs to a
+// FileTableTformJobAction at construction time must accord with the tforms
+// added later - some tforms "consume" inputs. For example, "replace column"
+// needs another file to read the replacement column from.
+class FileTableTformJobAction : public JobAction {
+  void anchor() override;
+
+public:
+  struct Tform {
+    enum Kind { EXTRACT, EXTRACT_DROP_TITLE, REPLACE };
+
+    Tform() = default;
+    Tform(Kind K, std::initializer_list<StringRef> Args) : TheKind(K) {
+      for (auto A : Args)
+        TheArgs.emplace_back(A.str());
+    }
+
+    Kind TheKind;
+    SmallVector<std::string, 2> TheArgs;
+  };
+
+  FileTableTformJobAction(Action *Input, types::ID OutputType);
+  FileTableTformJobAction(ActionList &Inputs, types::ID OutputType);
+
+  // Deletes all columns except the one with given name.
+  void addExtractColumnTform(StringRef ColumnName, bool WithColTitle = true);
+
+  // Replaces a column with title <From> in this table with a column with title
+  // <To> from another file table passed as input to this action.
+  void addReplaceColumnTform(StringRef From, StringRef To);
+
+  static bool classof(const Action *A) {
+    return A->getKind() == FileTableTformJobClass;
+  }
+
+  const ArrayRef<Tform> getTforms() const { return Tforms; }
+
+private:
+  SmallVector<Tform, 2> Tforms; // transformation actions requested
+};
+
 } // namespace driver
 } // namespace clang
 

clang/include/clang/Driver/ToolChain.h

@@ -147,6 +147,7 @@ class ToolChain {
   mutable std::unique_ptr<Tool> SYCLPostLink;
   mutable std::unique_ptr<Tool> PartialLink;
   mutable std::unique_ptr<Tool> BackendCompiler;
+  mutable std::unique_ptr<Tool> FileTableTform;
 
   Tool *getClang() const;
   Tool *getFlang() const;
@@ -161,6 +162,7 @@ class ToolChain {
   Tool *getSYCLPostLink() const;
   Tool *getPartialLink() const;
   Tool *getBackendCompiler() const;
+  Tool *getTableTform() const;
 
   mutable std::unique_ptr<SanitizerArgs> SanitizerArguments;
   mutable std::unique_ptr<XRayArgs> XRayArguments;

clang/include/clang/Driver/Types.def

@@ -103,7 +103,7 @@ TYPE("spirv",                    SPIRV,        INVALID,         "spv",    phases
 TYPE("sycl-header",              SYCL_Header,  INVALID,         "h",      phases::Compile, phases::Backend, phases::Assemble, phases::Link)
 TYPE("sycl-fatbin",              SYCL_FATBIN,  INVALID,         nullptr,  phases::Compile, phases::Backend, phases::Assemble, phases::Link)
 TYPE("tempfilelist",             Tempfilelist, INVALID,         "txt",    phases::Compile, phases::Backend, phases::Assemble, phases::Link)
-TYPE("tempentriesfilelist",      TempEntriesfilelist, INVALID,  "txt",    phases::Compile, phases::Backend, phases::Assemble, phases::Link)
+TYPE("tempfiletable",            Tempfiletable,INVALID,         "table",  phases::Compile, phases::Backend, phases::Assemble, phases::Link)
 TYPE("tempAOCOfilelist",         TempAOCOfilelist, INVALID,     "txt",    phases::Compile, phases::Backend, phases::Assemble, phases::Link)
 TYPE("archive",                  Archive,      INVALID,         "a",      phases::Compile, phases::Backend, phases::Assemble, phases::Link)
 TYPE("wholearchive",             WholeArchive, INVALID,         "a",      phases::Compile, phases::Backend, phases::Assemble, phases::Link)

clang/lib/Driver/Action.cpp

@@ -53,6 +53,8 @@ const char *Action::getClassName(ActionClass AC) {
     return "partial-link";
   case BackendCompileJobClass:
     return "backend-compiler";
+  case FileTableTformJobClass:
+    return "file-table-tform";
   }
 
   llvm_unreachable("invalid class");
@@ -473,3 +475,23 @@ BackendCompileJobAction::BackendCompileJobAction(ActionList &Inputs,
 BackendCompileJobAction::BackendCompileJobAction(Action *Input,
                                                  types::ID Type)
     : JobAction(BackendCompileJobClass, Input, Type) {}
+
+void FileTableTformJobAction::anchor() {}
+
+FileTableTformJobAction::FileTableTformJobAction(Action *Input, types::ID Type)
+    : JobAction(FileTableTformJobClass, Input, Type) {}
+
+FileTableTformJobAction::FileTableTformJobAction(ActionList &Inputs,
+                                                 types::ID Type)
+    : JobAction(FileTableTformJobClass, Inputs, Type) {}
+
+void FileTableTformJobAction::addExtractColumnTform(StringRef ColumnName,
+                                                    bool WithColTitle) {
+  auto K = WithColTitle ? Tform::EXTRACT : Tform::EXTRACT_DROP_TITLE;
+  Tforms.emplace_back(Tform(K, {ColumnName}));
+}
+
+void FileTableTformJobAction::addReplaceColumnTform(StringRef From,
+                                                    StringRef To) {
+  Tforms.emplace_back(Tform(Tform::REPLACE, {From, To}));
+}

clang/lib/Driver/Compilation.cpp

@@ -23,6 +23,7 @@
 #include "llvm/Option/OptSpecifier.h"
 #include "llvm/Option/Option.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/SimpleTable.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
 #include <fstream>
@@ -136,12 +137,29 @@ bool Compilation::CleanupFileList(const TempFileList &Files,
     // Temporary file lists contain files that need to be cleaned. The
     // file containing the information is also removed
     if (File.second == types::TY_Tempfilelist ||
-        File.second == types::TY_TempEntriesfilelist) {
-      std::ifstream ListFile(File.first);
-      if (ListFile) {
-        // These are temporary files and need to be removed.
+        File.second == types::TY_Tempfiletable) {
+      // These are temporary files and need to be removed.
+      bool IsTable = File.second == types::TY_Tempfiletable;
+
+      if (IsTable) {
+        if (llvm::sys::fs::exists(File.first)) {
+          auto T = llvm::util::SimpleTable::read(File.first);
+          if (!T) {
+            Success = false;
+            continue;
+          }
+          std::vector<std::string> TmpFileNames;
+          T->get()->linearize(TmpFileNames);
+
+          for (const auto &TmpFileName : TmpFileNames) {
+            if (!TmpFileName.empty())
+              Success &= CleanupFile(TmpFileName.c_str(), IssueErrors);
+          }
+        }
+      } else {
+        std::ifstream ListFile(File.first);
         std::string TmpFileName;
-        while(std::getline(ListFile, TmpFileName) && !TmpFileName.empty())
+        while (std::getline(ListFile, TmpFileName) && !TmpFileName.empty())
           Success &= CleanupFile(TmpFileName.c_str(), IssueErrors);
       }
     }