Skip to content

Commit e3dc0a4

Browse files
jmmartinezjmmartinez
committed
[Comgr][Cache] Cache unbundle of compressed bundles
1 parent 6ab9379 commit e3dc0a4

7 files changed

+316
-49
lines changed

‎amd/comgr/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ option(COMGR_BUILD_SHARED_LIBS "Build the shared library"
7070

7171
set(SOURCES
7272
src/comgr-cache.cpp
73+
src/comgr-cache-bundler-command.cpp
7374
src/comgr-cache-command.cpp
7475
src/comgr-clang-command.cpp
7576
src/comgr-compiler.cpp

‎amd/comgr/src/comgr-cache-bundler-command.cpp

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
/*******************************************************************************
2+
*
3+
* University of Illinois/NCSA
4+
* Open Source License
5+
*
6+
* Copyright (c) 2003-2017 University of Illinois at Urbana-Champaign.
7+
* Modifications (c) 2018 Advanced Micro Devices, Inc.
8+
* All rights reserved.
9+
*
10+
* Permission is hereby granted, free of charge, to any person obtaining a copy
11+
* of this software and associated documentation files (the "Software"), to deal
12+
* with the Software without restriction, including without limitation the
13+
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
14+
* sell copies of the Software, and to permit persons to whom the Software is
15+
* furnished to do so, subject to the following conditions:
16+
*
17+
* * Redistributions of source code must retain the above copyright notice,
18+
* this list of conditions and the following disclaimers.
19+
*
20+
* * Redistributions in binary form must reproduce the above copyright
21+
* notice, this list of conditions and the following disclaimers in the
22+
* documentation and/or other materials provided with the distribution.
23+
*
24+
* * Neither the names of the LLVM Team, University of Illinois at
25+
* Urbana-Champaign, nor the names of its contributors may be used to
26+
* endorse or promote products derived from this Software without specific
27+
* prior written permission.
28+
*
29+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
30+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
31+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
32+
* CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
33+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
34+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH
35+
* THE SOFTWARE.
36+
*
37+
******************************************************************************/
38+
39+
#include <comgr-cache-bundler-command.h>
40+
41+
#include <clang/Driver/OffloadBundler.h>
42+
#include <llvm/BinaryFormat/Magic.h>
43+
44+
namespace COMGR {
45+
using namespace llvm;
46+
using namespace clang;
47+
48+
using SizeFieldType = uint32_t;
49+
50+
bool UnbundleCommand::canCache() const {
51+
// The header format for AR files is not the same as object files
52+
if (Kind == AMD_COMGR_DATA_KIND_AR_BUNDLE)
53+
return false;
54+
55+
StringRef InputFilename = Config.InputFileNames.front();
56+
file_magic Magic;
57+
if (identify_magic(InputFilename, Magic))
58+
return false;
59+
60+
// Check the input file magic. Handle only compressed bundles
61+
// It's not worth to cache other types of bundles
62+
return Magic == file_magic::offload_bundle_compressed;
63+
}
64+
65+
Error UnbundleCommand::writeExecuteOutput(StringRef CachedBuffer) {
66+
for (StringRef OutputFilename : Config.OutputFileNames) {
67+
SizeFieldType OutputFileSize;
68+
if (CachedBuffer.size() < sizeof(OutputFileSize))
69+
return createStringError(std::errc::invalid_argument,
70+
"Not enough bytes to read output file size");
71+
memcpy(&OutputFileSize, CachedBuffer.data(), sizeof(OutputFileSize));
72+
CachedBuffer = CachedBuffer.drop_front(sizeof(OutputFileSize));
73+
74+
if (CachedBuffer.size() < OutputFileSize)
75+
return createStringError(std::errc::invalid_argument,
76+
"Not enough bytes to read output file contents");
77+
78+
StringRef OutputFileContents = CachedBuffer.substr(0, OutputFileSize);
79+
CachedBuffer = CachedBuffer.drop_front(OutputFileSize);
80+
81+
if (Error Err = CachedCommandAdaptor::writeUniqueExecuteOutput(
82+
OutputFilename, OutputFileContents))
83+
return Err;
84+
}
85+
86+
if (!CachedBuffer.empty())
87+
return createStringError(std::errc::invalid_argument,
88+
"Bytes in cache entry not used for the output");
89+
return Error::success();
90+
}
91+
92+
Expected<StringRef> UnbundleCommand::readExecuteOutput() {
93+
size_t OutputSize = 0;
94+
for (StringRef OutputFilename : Config.OutputFileNames) {
95+
auto MaybeOneOutput =
96+
CachedCommandAdaptor::readUniqueExecuteOutput(OutputFilename);
97+
if (!MaybeOneOutput)
98+
return MaybeOneOutput.takeError();
99+
100+
const MemoryBuffer &OneOutputBuffer = **MaybeOneOutput;
101+
SizeFieldType OneOutputFileSize = OneOutputBuffer.getBufferSize();
102+
103+
OutputBuffer.resize_for_overwrite(OutputSize + sizeof(OneOutputFileSize) +
104+
OneOutputFileSize);
105+
106+
memcpy(OutputBuffer.data() + OutputSize, &OneOutputFileSize,
107+
sizeof(OneOutputFileSize));
108+
OutputSize += sizeof(OneOutputFileSize);
109+
memcpy(OutputBuffer.data() + OutputSize, OneOutputBuffer.getBufferStart(),
110+
OneOutputFileSize);
111+
OutputSize += OneOutputFileSize;
112+
}
113+
return OutputBuffer;
114+
}
115+
116+
amd_comgr_status_t UnbundleCommand::execute(raw_ostream &LogS) {
117+
assert(Config.InputFileNames.size() == 1);
118+
119+
OffloadBundler Bundler(Config);
120+
121+
switch (Kind) {
122+
case AMD_COMGR_DATA_KIND_BC_BUNDLE:
123+
case AMD_COMGR_DATA_KIND_OBJ_BUNDLE: {
124+
if (Error Err = Bundler.UnbundleFiles()) {
125+
logAllUnhandledErrors(std::move(Err), LogS, "Unbundle Error: ");
126+
return AMD_COMGR_STATUS_ERROR;
127+
}
128+
break;
129+
}
130+
case AMD_COMGR_DATA_KIND_AR_BUNDLE: {
131+
if (Error Err = Bundler.UnbundleArchive()) {
132+
logAllUnhandledErrors(std::move(Err), LogS, "Unbundle Archives Error: ");
133+
return AMD_COMGR_STATUS_ERROR;
134+
}
135+
break;
136+
}
137+
default:
138+
llvm_unreachable("invalid bundle type");
139+
}
140+
141+
return AMD_COMGR_STATUS_SUCCESS;
142+
}
143+
144+
CachedCommandAdaptor::ActionClass UnbundleCommand::getClass() const {
145+
return clang::driver::Action::OffloadUnbundlingJobClass;
146+
}
147+
148+
void UnbundleCommand::addOptionsIdentifier(HashAlgorithm &H) const {
149+
H.update(Config.TargetNames.size());
150+
for (StringRef Target : Config.TargetNames) {
151+
CachedCommandAdaptor::addString(H, Target);
152+
}
153+
}
154+
155+
Error UnbundleCommand::addInputIdentifier(HashAlgorithm &H) const {
156+
StringRef InputFilename = Config.InputFileNames.front();
157+
158+
ErrorOr<std::unique_ptr<MemoryBuffer>> MaybeInputBuffer =
159+
MemoryBuffer::getFile(InputFilename);
160+
if (!MaybeInputBuffer) {
161+
std::error_code EC = MaybeInputBuffer.getError();
162+
return createStringError(EC, Twine("Failed to open ") + InputFilename +
163+
" : " + EC.message() + "\n");
164+
}
165+
166+
MemoryBuffer &InputBuffer = **MaybeInputBuffer;
167+
168+
using Header = CompressedOffloadBundle::CompressedBundleHeader;
169+
Expected<Header> MaybeHeader = Header::tryParse(InputBuffer.getBuffer());
170+
if (!MaybeHeader)
171+
return MaybeHeader.takeError();
172+
173+
// The hash represents the contents of the bundle. Extracting the same
174+
// contents should give the same result, regardless of the compression
175+
// algorithm or header version. Since the hash used by the offload bundler is
176+
// not a cryptographic hash, we also add the uncompressed file size.
177+
H.update(MaybeHeader->Hash);
178+
H.update(MaybeHeader->UncompressedFileSize);
179+
return Error::success();
180+
}
181+
182+
} // namespace COMGR

‎amd/comgr/src/comgr-cache-bundler-command.h

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/*******************************************************************************
2+
*
3+
* University of Illinois/NCSA
4+
* Open Source License
5+
*
6+
* Copyright (c) 2018 Advanced Micro Devices, Inc. All Rights Reserved.
7+
*
8+
* Permission is hereby granted, free of charge, to any person obtaining a copy
9+
* of this software and associated documentation files (the "Software"), to deal
10+
* with the Software without restriction, including without limitation the
11+
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
12+
* sell copies of the Software, and to permit persons to whom the Software is
13+
* furnished to do so, subject to the following conditions:
14+
*
15+
* * Redistributions of source code must retain the above copyright notice,
16+
* this list of conditions and the following disclaimers.
17+
*
18+
* * Redistributions in binary form must reproduce the above copyright
19+
* notice, this list of conditions and the following disclaimers in the
20+
* documentation and/or other materials provided with the distribution.
21+
*
22+
* * Neither the names of Advanced Micro Devices, Inc. nor the names of its
23+
* contributors may be used to endorse or promote products derived from
24+
* this Software without specific prior written permission.
25+
*
26+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
29+
* CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
30+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
31+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH
32+
* THE SOFTWARE.
33+
*
34+
******************************************************************************/
35+
36+
#ifndef COMGR_CACHE_BUNDLER_COMMAND_H
37+
#define COMGR_CACHE_BUNDLER_COMMAND_H
38+
39+
#include <comgr-cache-command.h>
40+
41+
namespace clang {
42+
class OffloadBundlerConfig;
43+
} // namespace clang
44+
45+
namespace COMGR {
46+
class UnbundleCommand final : public CachedCommandAdaptor {
47+
private:
48+
amd_comgr_data_kind_t Kind;
49+
const clang::OffloadBundlerConfig &Config;
50+
51+
// To avoid copies, store the output of execute, such that readExecuteOutput
52+
// can return a reference.
53+
llvm::SmallString<64> OutputBuffer;
54+
55+
public:
56+
UnbundleCommand(amd_comgr_data_kind_t Kind,
57+
const clang::OffloadBundlerConfig &Config)
58+
: Kind(Kind), Config(Config) {}
59+
60+
bool canCache() const override;
61+
llvm::Error writeExecuteOutput(llvm::StringRef CachedBuffer) override;
62+
llvm::Expected<llvm::StringRef> readExecuteOutput() override;
63+
amd_comgr_status_t execute(llvm::raw_ostream &LogS) override;
64+
65+
~UnbundleCommand() override = default;
66+
67+
protected:
68+
ActionClass getClass() const override;
69+
void addOptionsIdentifier(HashAlgorithm &) const override;
70+
llvm::Error addInputIdentifier(HashAlgorithm &) const override;
71+
};
72+
} // namespace COMGR
73+
74+
#endif

‎amd/comgr/src/comgr-cache-command.cpp

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,4 +98,39 @@ CachedCommandAdaptor::getIdentifier() const {
9898
toHex(H.final(), true, Id);
9999
return Id;
100100
}
101+
102+
llvm::Error
103+
CachedCommandAdaptor::writeUniqueExecuteOutput(StringRef OutputFilename,
104+
StringRef CachedBuffer) {
105+
std::error_code EC;
106+
raw_fd_ostream Out(OutputFilename, EC);
107+
if (EC) {
108+
Error E = createStringError(EC, Twine("Failed to open ") + OutputFilename +
109+
" : " + EC.message() + "\n");
110+
return E;
111+
}
112+
113+
Out.write(CachedBuffer.data(), CachedBuffer.size());
114+
Out.close();
115+
if (Out.has_error()) {
116+
Error E = createStringError(EC, Twine("Failed to write ") + OutputFilename +
117+
" : " + EC.message() + "\n");
118+
return E;
119+
}
120+
121+
return Error::success();
122+
}
123+
124+
Expected<std::unique_ptr<MemoryBuffer>>
125+
CachedCommandAdaptor::readUniqueExecuteOutput(StringRef OutputFilename) {
126+
ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr =
127+
MemoryBuffer::getFile(OutputFilename);
128+
if (!MBOrErr) {
129+
std::error_code EC = MBOrErr.getError();
130+
return createStringError(EC, Twine("Failed to open ") + OutputFilename +
131+
" : " + EC.message() + "\n");
132+
}
133+
134+
return std::move(*MBOrErr);
135+
}
101136
} // namespace COMGR

‎amd/comgr/src/comgr-cache-command.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,12 @@ class CachedCommandAdaptor {
6969
static void addString(HashAlgorithm &H, llvm::StringRef S);
7070
static std::optional<size_t> searchComgrTmpModel(llvm::StringRef S);
7171

72+
// helper since several command types just write to a single output file
73+
static llvm::Error writeUniqueExecuteOutput(llvm::StringRef OutputFilename,
74+
llvm::StringRef CachedBuffer);
75+
static llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>>
76+
readUniqueExecuteOutput(llvm::StringRef OutputFilename);
77+
7278
protected:
7379
virtual ActionClass getClass() const = 0;
7480
virtual void addOptionsIdentifier(HashAlgorithm &) const = 0;

‎amd/comgr/src/comgr-clang-command.cpp

Lines changed: 7 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -158,35 +158,16 @@ bool ClangCommand::canCache() const {
158158

159159
Error ClangCommand::writeExecuteOutput(StringRef CachedBuffer) {
160160
StringRef OutputFilename = Command.getOutputFilenames().front();
161-
std::error_code EC;
162-
raw_fd_ostream Out(OutputFilename, EC);
163-
if (EC) {
164-
Error E = createStringError(EC, Twine("Failed to open ") + OutputFilename +
165-
" : " + EC.message() + "\n");
166-
return E;
167-
}
168-
169-
Out.write(CachedBuffer.data(), CachedBuffer.size());
170-
Out.close();
171-
if (Out.has_error()) {
172-
Error E = createStringError(EC, Twine("Failed to write ") + OutputFilename +
173-
" : " + EC.message() + "\n");
174-
return E;
175-
}
176-
177-
return Error::success();
161+
return CachedCommandAdaptor::writeUniqueExecuteOutput(OutputFilename,
162+
CachedBuffer);
178163
}
179164

180165
Expected<StringRef> ClangCommand::readExecuteOutput() {
181-
StringRef OutputFilename = Command.getOutputFilenames().front();
182-
ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr =
183-
MemoryBuffer::getFile(OutputFilename);
184-
if (!MBOrErr) {
185-
std::error_code EC = MBOrErr.getError();
186-
return createStringError(EC, Twine("Failed to open ") + OutputFilename +
187-
" : " + EC.message() + "\n");
188-
}
189-
Output = std::move(*MBOrErr);
166+
auto MaybeBuffer = CachedCommandAdaptor::readUniqueExecuteOutput(
167+
Command.getOutputFilenames().front());
168+
if (!MaybeBuffer)
169+
return MaybeBuffer.takeError();
170+
Output = std::move(*MaybeBuffer);
190171
return Output->getBuffer();
191172
}
192173

0 commit comments

Comments
 (0)