[mlir][transform] Support symlinks in module loading. Reorganize tests. (#69329)

A recent commit (#69190) broke the bazel builds. Turns out that Bazel
uses symlinks for providing the test files, which the path expansion of
the module loading mechanism did not handle correctly. This PR fixes
that.

It also reorganizes the tests better: It puts all `.mlir` files that are
included by some other test into a common `include` folder. This greatly
simplifies the definition of the dependencies between the different
`.mlir` files in Bazel's `BUILD` file. The commit also adds a comment to
all included files why these aren't tested themselves direclty and uses
the `%{fs-sep}` expansion for paths more consistently. Finally, it
uncomments all but one of the tests excluded in Bazel because they seem
to run now. (The remaining one includes a file that it itself a test, so
it would have to live *in* and *outside* of the `include` folder.)

Author: ingomueller-net

mlir/lib/Dialect/Transform/Transforms/TransformInterpreterUtils.cpp

@@ -61,7 +61,8 @@ LogicalResult transform::detail::expandPathsToMLIRFiles(
          it != itEnd && !ec; it.increment(ec)) {
       const std::string &fileName = it->path();
 
-      if (it->type() != llvm::sys::fs::file_type::regular_file) {
+      if (it->type() != llvm::sys::fs::file_type::regular_file &&
+          it->type() != llvm::sys::fs::file_type::symlink_file) {
         LLVM_DEBUG(DBGS() << "  Skipping non-regular file '" << fileName
                           << "'\n");
         continue;

mlir/test/Dialect/Transform/Library/lower-to-llvm.mlir renamed to mlir/test/Dialect/Transform/include/Library/lower-to-llvm.mlir

@@ -1,4 +1,5 @@
 // RUN: mlir-opt %s
+// No need to check anything else than parsing here, this is being used by another test as data.
 
 /// Schedule to lower to LLVM.
 module @lower_module_to_llvm attributes { transform.with_named_sequence } {

mlir/test/Dialect/Transform/test-interpreter-external-concurrent-source.mlir renamed to mlir/test/Dialect/Transform/include/test-interpreter-external-concurrent-source.mlir

@@ -1,4 +1,5 @@
 // RUN: mlir-opt %s
+// No need to check anything else than parsing here, this is being used by another test as data.
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence private @private_helper(%arg0: !transform.any_op {transform.readonly}) {

mlir/test/Dialect/Transform/test-interpreter-external-source.mlir renamed to mlir/test/Dialect/Transform/include/test-interpreter-external-source.mlir

@@ -1,4 +1,5 @@
 // RUN: mlir-opt %s
+// No need to check anything else than parsing here, this is being used by another test as data.
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @print_message(%arg0: !transform.any_op {transform.readonly})

mlir/test/Dialect/Transform/test-interpreter-external-symbol-def-invalid.mlir renamed to mlir/test/Dialect/Transform/include/test-interpreter-external-symbol-def-invalid.mlir

@@ -1,5 +1,5 @@
 // RUN: mlir-opt %s \
-// RUN:   -transform-preload-library=transform-library-paths=%p%{fs-sep}test-interpreter-library \
+// RUN:   -transform-preload-library=transform-library-paths=%p%{fs-sep}include%{fs-sep}test-interpreter-library \
 // RUN:   -transform-interpreter=entry-point=private_helper \
 // RUN:   -split-input-file -verify-diagnostics
 

mlir/test/Dialect/Transform/test-interpreter-library/definitions-self-contained.mlir renamed to mlir/test/Dialect/Transform/include/test-interpreter-library/definitions-self-contained.mlir

@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --pass-pipeline="builtin.module(func.func(test-transform-dialect-interpreter{transform-file-name=%p/test-interpreter-external-concurrent-source.mlir}))" \
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(func.func(test-transform-dialect-interpreter{transform-file-name=%p%{fs-sep}include%{fs-sep}test-interpreter-external-concurrent-source.mlir}))" \
 // RUN:             --verify-diagnostics
 
 // Exercising the pass on multiple functions of different lengths that may be

mlir/test/Dialect/Transform/test-interpreter-library/definitions-with-unresolved.mlir renamed to mlir/test/Dialect/Transform/include/test-interpreter-library/definitions-with-unresolved.mlir

@@ -1,7 +1,7 @@
-// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-file-name=%p/test-interpreter-external-symbol-decl.mlir transform-library-paths=%p/test-interpreter-library/definitions-self-contained.mlir})" \
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-file-name=%p%{fs-sep}test-interpreter-external-symbol-decl.mlir transform-library-paths=%p%{fs-sep}include%{fs-sep}test-interpreter-library/definitions-self-contained.mlir})" \
 // RUN:             --verify-diagnostics
 
-// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-file-name=%p/test-interpreter-external-symbol-decl.mlir transform-library-paths=%p/test-interpreter-library/definitions-self-contained.mlir}, test-transform-dialect-interpreter{transform-file-name=%p/test-interpreter-external-symbol-decl.mlir transform-library-paths=%p/test-interpreter-library/definitions-self-contained.mlir})" \
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-file-name=%p%{fs-sep}test-interpreter-external-symbol-decl.mlir transform-library-paths=%p%{fs-sep}include%{fs-sep}test-interpreter-library/definitions-self-contained.mlir}, test-transform-dialect-interpreter{transform-file-name=%p%{fs-sep}test-interpreter-external-symbol-decl.mlir transform-library-paths=%p%{fs-sep}include%{fs-sep}test-interpreter-library/definitions-self-contained.mlir})" \
 // RUN:             --verify-diagnostics
 
 // The external transform script has a declaration to the named sequence @foo,

mlir/test/Dialect/Transform/preload-library.mlir

@@ -1,10 +1,10 @@
-// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-library-paths=%p%{fs-sep}test-interpreter-library})" \
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-library-paths=%p%{fs-sep}include%{fs-sep}test-interpreter-library})" \
 // RUN:             --verify-diagnostics --split-input-file | FileCheck %s
 
-// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-library-paths=%p%{fs-sep}test-interpreter-library/definitions-self-contained.mlir,%p%{fs-sep}test-interpreter-library/definitions-with-unresolved.mlir})" \
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-library-paths=%p%{fs-sep}include%{fs-sep}test-interpreter-library/definitions-self-contained.mlir,%p%{fs-sep}include%{fs-sep}test-interpreter-library/definitions-with-unresolved.mlir})" \
 // RUN:             --verify-diagnostics --split-input-file | FileCheck %s
 
-// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-library-paths=%p%{fs-sep}test-interpreter-library}, test-transform-dialect-interpreter)" \
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-library-paths=%p%{fs-sep}include%{fs-sep}test-interpreter-library}, test-transform-dialect-interpreter)" \
 // RUN:             --verify-diagnostics --split-input-file | FileCheck %s
 
 // The definition of the @foo named sequence is provided in another file. It

mlir/test/Dialect/Transform/test-interpreter-external-concurrent.mlir

@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-library-paths=%p/test-interpreter-external-symbol-def-invalid.mlir}, test-transform-dialect-interpreter)" \
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-library-paths=%p%{fs-sep}include%{fs-sep}test-interpreter-external-symbol-def-invalid.mlir}, test-transform-dialect-interpreter)" \
 // RUN:             --verify-diagnostics --split-input-file
 
 // The definition of the @print_message named sequence is provided in another file. It

mlir/test/Dialect/Transform/test-interpreter-external-symbol-decl-and-schedule.mlir

@@ -1,7 +1,7 @@
-// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-library-paths=%p/test-interpreter-library/definitions-self-contained.mlir})" \
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-library-paths=%p%{fs-sep}include%{fs-sep}test-interpreter-library/definitions-self-contained.mlir})" \
 // RUN:             --verify-diagnostics --split-input-file | FileCheck %s
 
-// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-library-paths=%p/test-interpreter-library/definitions-self-contained.mlir}, test-transform-dialect-interpreter)" \
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-library-paths=%p%{fs-sep}include%{fs-sep}test-interpreter-library/definitions-self-contained.mlir}, test-transform-dialect-interpreter)" \
 // RUN:             --verify-diagnostics --split-input-file | FileCheck %s
 
 // The definition of the @print_message named sequence is provided in another

mlir/test/Dialect/Transform/test-interpreter-external-symbol-decl-dir.mlir

@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-file-name=%p/test-interpreter-external-source.mlir})" \
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(test-transform-dialect-interpreter{transform-file-name=%p%{fs-sep}include%{fs-sep}test-interpreter-external-source.mlir})" \
 // RUN:             --verify-diagnostics
 
 // The schedule in the separate file emits remarks at the payload root.

mlir/test/Dialect/Transform/test-interpreter-external-symbol-decl-invalid.mlir

@@ -18,27 +18,16 @@ package(default_visibility = ["//visibility:public"])
         ] + glob([
             "IRDL/*.irdl.mlir",
             "LLVM/*-symbol-def.mlir",
-            "Transform/*-source.mlir",
-            "Transform/*-symbol-def.mlir",
-            "Transform/*-symbol-decl-and-schedule.mlir",
-            "Transform/Library/*.mlir",
-            "Transform/test-interpreter-library/*.mlir",
+            "Transform/include/**/*.mlir",
         ]),
     )
     for src in glob(
         include = ["**/*.mlir"],
         exclude = [
             "IRDL/*.irdl.mlir",
             "LLVM/*-symbol-def.mlir",
-            "Transform/*-source.mlir",
-            "Transform/*-symbol-def.mlir",
             "Transform/*-symbol-decl-and-schedule.mlir",
-            "Transform/*-symbol-decl-dir.mlir",
-            "Transform/*-symbol-decl-invalid.mlir",
-            "Transform/Library/*.mlir",
-            "Transform/preload-library.mlir",
-            "Transform/test-interpreter-library/*.mlir",
-            "Transform/test-repro-dump.mlir",
+            "Transform/include/**/*.mlir",
         ],
     )
 ]