Skip to content

Commit 89f689e

Browse files
LU-JOHNLU-JOHN
authored
[SYCL][clang-offload-bundler] Add support for BC files in archives. (#11034)
Add support for -list, -check-section, and -unbundle for bundled BC files in archives. --------- Signed-off-by: Lu, John <[email protected]>
1 parent e213fe2 commit 89f689e

File tree

4 files changed

+354
-44
lines changed

4 files changed

+354
-44
lines changed

‎clang/lib/Driver/OffloadBundler.cpp

Lines changed: 67 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1059,39 +1059,49 @@ class ArchiveFileHandler final : public FileHandler {
10591059
for (auto &C : Ar->children(Err)) {
10601060
++ChildIndex;
10611061
auto BinOrErr = C.getAsBinary();
1062+
1063+
std::unique_ptr<FileHandler> FH{nullptr};
1064+
std::unique_ptr<MemoryBuffer> Buf{nullptr};
1065+
10621066
if (!BinOrErr) {
10631067
if (auto Err = isNotObjectErrorInvalidFileType(BinOrErr.takeError()))
10641068
return Err;
1065-
continue;
1066-
}
10671069

1068-
auto &Bin = BinOrErr.get();
1069-
if (!Bin->isObject())
1070-
continue;
1070+
// Handle bundled BC Files
1071+
FH = std::make_unique<BinaryFileHandler>(BundlerConfig);
1072+
auto MR = C.getMemoryBufferRef();
1073+
assert(MR);
1074+
Buf = MemoryBuffer::getMemBuffer(*MR, false);
1075+
} else {
1076+
auto &Bin = BinOrErr.get();
1077+
if (!Bin->isObject())
1078+
continue;
10711079

1072-
auto CheckOrErr = CheckIfObjectFileContainsExcludedTargets(C);
1073-
if (!CheckOrErr)
1074-
return CheckOrErr.takeError();
1080+
auto CheckOrErr = CheckIfObjectFileContainsExcludedTargets(C);
1081+
if (!CheckOrErr)
1082+
return CheckOrErr.takeError();
10751083

1076-
if (*CheckOrErr) {
1077-
LLVM_DEBUG(outs() << "Add child to ban list. Index: " << ChildIndex
1078-
<< "\n");
1079-
ExcludedChildIndexes.emplace(ChildIndex);
1080-
}
1084+
if (*CheckOrErr) {
1085+
LLVM_DEBUG(outs()
1086+
<< "Add child to ban list. Index: " << ChildIndex << "\n");
1087+
ExcludedChildIndexes.emplace(ChildIndex);
1088+
}
10811089

1082-
auto Obj = std::unique_ptr<ObjectFile>(cast<ObjectFile>(Bin.release()));
1083-
auto Buf = MemoryBuffer::getMemBuffer(Obj->getMemoryBufferRef(), false);
1090+
auto Obj = std::unique_ptr<ObjectFile>(cast<ObjectFile>(Bin.release()));
1091+
Buf = MemoryBuffer::getMemBuffer(Obj->getMemoryBufferRef(), false);
10841092

1085-
// Collect the list of bundles from the object.
1086-
ObjectFileHandler OFH(std::move(Obj), BundlerConfig);
1087-
if (Error Err = OFH.ReadHeader(*Buf))
1093+
FH = std::make_unique<ObjectFileHandler>(std::move(Obj), BundlerConfig);
1094+
}
1095+
1096+
// Collect the list of bundles from the object or bundled BC file.
1097+
if (Error Err = FH->ReadHeader(*Buf))
10881098
return Err;
1089-
Expected<std::optional<StringRef>> NameOrErr = OFH.ReadBundleStart(*Buf);
1099+
Expected<std::optional<StringRef>> NameOrErr = FH->ReadBundleStart(*Buf);
10901100
if (!NameOrErr)
10911101
return NameOrErr.takeError();
10921102
while (*NameOrErr) {
10931103
++Bundles[**NameOrErr];
1094-
NameOrErr = OFH.ReadBundleStart(*Buf);
1104+
NameOrErr = FH->ReadBundleStart(*Buf);
10951105
if (!NameOrErr)
10961106
return NameOrErr.takeError();
10971107
}
@@ -1144,28 +1154,40 @@ class ArchiveFileHandler final : public FileHandler {
11441154
continue;
11451155
}
11461156

1157+
std::unique_ptr<FileHandler> FH{nullptr};
1158+
std::unique_ptr<MemoryBuffer> Buf{nullptr};
1159+
StringRef Ext("o");
1160+
if (BundlerConfig.FilesType == "aocr" ||
1161+
BundlerConfig.FilesType == "aocx")
1162+
Ext = BundlerConfig.FilesType;
1163+
11471164
auto BinOrErr = C.getAsBinary();
11481165
if (!BinOrErr) {
1166+
// Not a recognized binary file. Specifically not an object file
11491167
if (auto Err = isNotObjectErrorInvalidFileType(BinOrErr.takeError()))
11501168
return Err;
1151-
continue;
1152-
}
1153-
1154-
auto &Bin = BinOrErr.get();
1155-
if (!Bin->isObject())
1156-
continue;
1157-
1158-
auto Obj = std::unique_ptr<ObjectFile>(cast<ObjectFile>(Bin.release()));
1159-
auto Buf = MemoryBuffer::getMemBuffer(Obj->getMemoryBufferRef(), false);
11601169

1161-
auto ChildNameOrErr = C.getName();
1162-
if (!ChildNameOrErr)
1163-
return ChildNameOrErr.takeError();
1170+
if (BundlerConfig.FilesType == "aoo") {
1171+
// Handle bundled BC Files
1172+
Ext = "bc";
1173+
FH = std::make_unique<BinaryFileHandler>(BundlerConfig);
1174+
auto MR = C.getMemoryBufferRef();
1175+
assert(MR);
1176+
Buf = MemoryBuffer::getMemBuffer(*MR, false);
1177+
} else
1178+
continue;
1179+
} else {
1180+
auto &Bin = BinOrErr.get();
1181+
if (!Bin->isObject())
1182+
continue;
1183+
auto Obj = std::unique_ptr<ObjectFile>(cast<ObjectFile>(Bin.release()));
1184+
Buf = MemoryBuffer::getMemBuffer(Obj->getMemoryBufferRef(), false);
1185+
FH = std::make_unique<ObjectFileHandler>(std::move(Obj), BundlerConfig);
1186+
}
11641187

1165-
ObjectFileHandler OFH(std::move(Obj), BundlerConfig);
1166-
if (Error Err = OFH.ReadHeader(*Buf))
1188+
if (Error Err = FH->ReadHeader(*Buf))
11671189
return Err;
1168-
Expected<std::optional<StringRef>> NameOrErr = OFH.ReadBundleStart(*Buf);
1190+
Expected<std::optional<StringRef>> NameOrErr = FH->ReadBundleStart(*Buf);
11691191
if (!NameOrErr)
11701192
return NameOrErr.takeError();
11711193
while (*NameOrErr) {
@@ -1175,10 +1197,7 @@ class ArchiveFileHandler final : public FileHandler {
11751197
if (Mode == OutputType::FileList) {
11761198
// Create temporary file where the device part will be extracted to.
11771199
SmallString<128u> ChildFileName;
1178-
StringRef Ext("o");
1179-
if (BundlerConfig.FilesType == "aocr" ||
1180-
BundlerConfig.FilesType == "aocx")
1181-
Ext = BundlerConfig.FilesType;
1200+
11821201
auto EC = sys::fs::createTemporaryFile(TempFileNameBase, Ext,
11831202
ChildFileName);
11841203
if (EC)
@@ -1188,7 +1207,7 @@ class ArchiveFileHandler final : public FileHandler {
11881207
if (EC)
11891208
return createFileError(ChildFileName, EC);
11901209

1191-
if (Error Err = OFH.ReadBundle(ChildOS, *Buf))
1210+
if (Error Err = FH->ReadBundle(ChildOS, *Buf))
11921211
return Err;
11931212

11941213
if (ChildOS.has_error())
@@ -1199,13 +1218,17 @@ class ArchiveFileHandler final : public FileHandler {
11991218
OS << ChildFileName << "\n";
12001219
} else if (Mode == OutputType::Object) {
12011220
// Extract the bundle to the output file in single file mode.
1202-
if (Error Err = OFH.ReadBundle(OS, *Buf))
1221+
if (Error Err = FH->ReadBundle(OS, *Buf))
12031222
return Err;
12041223
} else if (Mode == OutputType::Archive) {
1224+
auto ChildNameOrErr = C.getName();
1225+
if (!ChildNameOrErr)
1226+
return ChildNameOrErr.takeError();
1227+
12051228
// Extract the bundle to a buffer.
12061229
SmallVector<char> Data;
12071230
raw_svector_ostream ChildOS{Data};
1208-
if (Error Err = OFH.ReadBundle(ChildOS, *Buf))
1231+
if (Error Err = FH->ReadBundle(ChildOS, *Buf))
12091232
return Err;
12101233

12111234
// Add new archive member.
@@ -1214,10 +1237,10 @@ class ArchiveFileHandler final : public FileHandler {
12141237
Member.Buf = MemoryBuffer::getMemBufferCopy(ChildOS.str(), Name);
12151238
Member.MemberName = Member.Buf->getBufferIdentifier();
12161239
}
1217-
if (Error Err = OFH.ReadBundleEnd(*Buf))
1240+
if (Error Err = FH->ReadBundleEnd(*Buf))
12181241
return Err;
12191242
}
1220-
NameOrErr = OFH.ReadBundleStart(*Buf);
1243+
NameOrErr = FH->ReadBundleStart(*Buf);
12211244
if (!NameOrErr)
12221245
return NameOrErr.takeError();
12231246
}

‎clang/test/Driver/clang-offload-bundler-bc-archive-support-linux.cpp

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
// Target "host-x86_64-unknown-linux-gnu" only works on Linux
2+
// REQUIRES: system-linux
3+
4+
// Ensure that bundled BC files in archives can work with:
5+
// TEST1: clang-offload-bundler -list
6+
// TEST2: clang-offload-bundler -check-section
7+
// TEST3: clang-offload-bundler -unbundle with single target
8+
// TEST4: clang-offload-bundler -unbundle with multiple targets
9+
//
10+
// In all these tests also ensure functionality with bundled object files still
11+
// works correctly.
12+
13+
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
14+
// Make bundled object with targets:
15+
// sycl-spir64-unknown-unknown
16+
// host-x86_64-unknown-linux-gnu
17+
// RUN: %clangxx -fsycl -c %s -o %t_bundled.o
18+
19+
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
20+
// Make three distinct BC files
21+
// RUN: %clangxx -fsycl -fsycl-device-only -DTYPE1 %s -o %t1.bc
22+
// RUN: %clangxx -fsycl -fsycl-device-only -DTYPE2 %s -o %t2.bc
23+
// RUN: %clangxx -fsycl -fsycl-device-only -DTYPE3 %s -o %t3.bc
24+
25+
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
26+
// Bundle BC files to different targets:
27+
// host-spir64-unknown-unknown
28+
// host-spir64_gen
29+
// host-spir64_x86_64
30+
// RUN: clang-offload-bundler -type=bc -targets=host-spir64-unknown-unknown -input=%t1.bc -output=%t1_bundled.bc
31+
// RUN: clang-offload-bundler -type=bc -targets=host-spir64_gen -input=%t2.bc -output=%t2_bundled.bc
32+
// RUN: clang-offload-bundler -type=bc -targets=host-spir64_x86_64 -input=%t3.bc -output=%t3_bundled.bc
33+
34+
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
35+
// Make archive with bundled BC and o files
36+
// RUN: rm -f %t_bundled.a
37+
// RUN: llvm-ar cr %t_bundled.a %t1_bundled.bc %t2_bundled.bc %t3_bundled.bc %t_bundled.o
38+
39+
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
40+
// TEST1
41+
// Check that -list with various archive types can find all targets
42+
// RUN: clang-offload-bundler -list -type=ao -input=%t_bundled.a > %t_list_ao.txt
43+
// RUN: clang-offload-bundler -list -type=aoo -input=%t_bundled.a > %t_list_aoo.txt
44+
// RUN: FileCheck --check-prefixes=CHECK-LIST < %t_list_ao.txt %s
45+
// RUN: FileCheck --check-prefixes=CHECK-LIST < %t_list_aoo.txt %s
46+
47+
// CHECK-LIST-DAG: sycl-spir64-unknown-unknown
48+
// CHECK-LIST-DAG: host-x86_64-unknown-linux-gnu
49+
// CHECK-LIST-DAG: host-spir64-unknown-unknown
50+
// CHECK-LIST-DAG: host-spir64_gen
51+
// CHECK-LIST-DAG: host-spir64_x86_64
52+
53+
// RUN: wc -l %t_list_ao.txt | FileCheck --check-prefixes=CHECK-LIST-LENGTH %s
54+
// RUN: wc -l %t_list_aoo.txt | FileCheck --check-prefixes=CHECK-LIST-LENGTH %s
55+
56+
// CHECK-LIST-LENGTH: 5
57+
58+
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
59+
// TEST2
60+
// Test -check-section
61+
// RUN: clang-offload-bundler -check-section -type=ao -input=%t_bundled.a -targets=sycl-spir64-unknown-unknown
62+
// RUN: clang-offload-bundler -check-section -type=ao -input=%t_bundled.a -targets=host-x86_64-unknown-linux-gnu
63+
// RUN: clang-offload-bundler -check-section -type=ao -input=%t_bundled.a -targets=host-spir64-unknown-unknown
64+
// RUN: clang-offload-bundler -check-section -type=ao -input=%t_bundled.a -targets=host-spir64_gen
65+
// RUN: clang-offload-bundler -check-section -type=ao -input=%t_bundled.a -targets=host-spir64_x86_64
66+
// RUN: not clang-offload-bundler -check-section -type=ao -input=%t_bundled.a -targets=sycl-spir64-unknown-unknown-a
67+
// RUN: not clang-offload-bundler -check-section -type=ao -input=%t_bundled.a -targets=host-x86_64-unknown-linux-gnu-b
68+
// RUN: not clang-offload-bundler -check-section -type=ao -input=%t_bundled.a -targets=host-spir64-unknown-unknown-c
69+
// RUN: not clang-offload-bundler -check-section -type=ao -input=%t_bundled.a -targets=host-spir64_gen-d
70+
// RUN: not clang-offload-bundler -check-section -type=ao -input=%t_bundled.a -targets=host-spir64_x86_64-e
71+
// RUN: clang-offload-bundler -check-section -type=aoo -input=%t_bundled.a -targets=sycl-spir64-unknown-unknown
72+
// RUN: clang-offload-bundler -check-section -type=aoo -input=%t_bundled.a -targets=host-x86_64-unknown-linux-gnu
73+
// RUN: clang-offload-bundler -check-section -type=aoo -input=%t_bundled.a -targets=host-spir64-unknown-unknown
74+
// RUN: clang-offload-bundler -check-section -type=aoo -input=%t_bundled.a -targets=host-spir64_gen
75+
// RUN: clang-offload-bundler -check-section -type=aoo -input=%t_bundled.a -targets=host-spir64_x86_64
76+
// RUN: not clang-offload-bundler -check-section -type=aoo -input=%t_bundled.a -targets=sycl-spir64-unknown-unknown-a
77+
// RUN: not clang-offload-bundler -check-section -type=aoo -input=%t_bundled.a -targets=host-x86_64-unknown-linux-gnu-b
78+
// RUN: not clang-offload-bundler -check-section -type=aoo -input=%t_bundled.a -targets=host-spir64-unknown-unknown-c
79+
// RUN: not clang-offload-bundler -check-section -type=aoo -input=%t_bundled.a -targets=host-spir64_gen-d
80+
// RUN: not clang-offload-bundler -check-section -type=aoo -input=%t_bundled.a -targets=host-spir64_x86_64-e
81+
82+
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
83+
// Unbundle object file to use as a reference result
84+
// RUN: clang-offload-bundler -unbundle -type=o -input=%t_bundled.o -targets=sycl-spir64-unknown-unknown -output=%t_unbundled_A.o
85+
// RUN: clang-offload-bundler -unbundle -type=o -input=%t_bundled.o -targets=host-x86_64-unknown-linux-gnu -output=%t_unbundled_B.o
86+
87+
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
88+
// TEST3
89+
// Test archive unbundling
90+
// RUN: clang-offload-bundler -unbundle -type=aoo -input=%t_bundled.a -targets=sycl-spir64-unknown-unknown -output=%t_list1.txt
91+
// RUN: clang-offload-bundler -unbundle -type=aoo -input=%t_bundled.a -targets=host-x86_64-unknown-linux-gnu -output=%t_list2.txt
92+
// RUN: clang-offload-bundler -unbundle -type=aoo -input=%t_bundled.a -targets=host-spir64-unknown-unknown -output=%t_list3.txt
93+
// RUN: clang-offload-bundler -unbundle -type=aoo -input=%t_bundled.a -targets=host-spir64_gen -output=%t_list4.txt
94+
// RUN: clang-offload-bundler -unbundle -type=aoo -input=%t_bundled.a -targets=host-spir64_x86_64 -output=%t_list5.txt
95+
96+
// RUN: cmp %t_unbundled_A.o `cat %t_list1.txt`
97+
// RUN: cmp %t_unbundled_B.o `cat %t_list2.txt`
98+
// RUN: cmp %t1.bc `cat %t_list3.txt`
99+
// RUN: cmp %t2.bc `cat %t_list4.txt`
100+
// RUN: cmp %t3.bc `cat %t_list5.txt`
101+
102+
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
103+
// TEST4
104+
// Test archive unbundling for multiple targets
105+
// RUN: clang-offload-bundler -unbundle -type=aoo -input=%t_bundled.a -targets=sycl-spir64-unknown-unknown,host-spir64_gen -output=%t_listA.txt -output=%t_listB.txt
106+
107+
// RUN: cmp %t_unbundled_A.o `cat %t_listA.txt`
108+
// RUN: cmp %t2.bc `cat %t_listB.txt`
109+
110+
#include <sycl/sycl.hpp>
111+
112+
SYCL_EXTERNAL int foo(int x) {
113+
114+
#ifdef TYPE1
115+
return x+13;
116+
#elif TYPE2
117+
return x+17;
118+
#elif TYPE3
119+
return x+23;
120+
#else
121+
return x+29;
122+
#endif
123+
}

0 commit comments

Comments
 (0)