Skip to content

Commit bebcbfb

Browse files
author
George Karpenkov
committed
[libFuzzer] Use custom allocators for STL containers in libFuzzer.
Avoids ODR violations causing spurious ASAN warnings. Differential Revision: https://reviews.llvm.org/D37086 llvm-svn: 311866
1 parent 4139502 commit bebcbfb

19 files changed

+113
-92
lines changed

‎compiler-rt/lib/fuzzer/FuzzerCorpus.h

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ struct InputInfo {
3535
size_t NumSuccessfullMutations = 0;
3636
bool MayDeleteFile = false;
3737
bool Reduced = false;
38-
std::vector<uint32_t> UniqFeatureSet;
38+
Vector<uint32_t> UniqFeatureSet;
3939
};
4040

4141
class InputCorpus {
@@ -71,7 +71,7 @@ class InputCorpus {
7171
bool empty() const { return Inputs.empty(); }
7272
const Unit &operator[] (size_t Idx) const { return Inputs[Idx]->U; }
7373
void AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile,
74-
const std::vector<uint32_t> &FeatureSet) {
74+
const Vector<uint32_t> &FeatureSet) {
7575
assert(!U.empty());
7676
if (FeatureDebug)
7777
Printf("ADD_TO_CORPUS %zd NF %zd\n", Inputs.size(), NumFeatures);
@@ -100,7 +100,7 @@ class InputCorpus {
100100
}
101101

102102
// Debug-only
103-
void PrintFeatureSet(const std::vector<uint32_t> &FeatureSet) {
103+
void PrintFeatureSet(const Vector<uint32_t> &FeatureSet) {
104104
if (!FeatureDebug) return;
105105
Printf("{");
106106
for (uint32_t Feature: FeatureSet)
@@ -256,11 +256,11 @@ class InputCorpus {
256256
}
257257
std::piecewise_constant_distribution<double> CorpusDistribution;
258258

259-
std::vector<double> Intervals;
260-
std::vector<double> Weights;
259+
Vector<double> Intervals;
260+
Vector<double> Weights;
261261

262262
std::unordered_set<std::string> Hashes;
263-
std::vector<InputInfo*> Inputs;
263+
Vector<InputInfo*> Inputs;
264264

265265
size_t NumAddedFeatures = 0;
266266
size_t NumUpdatedFeatures = 0;

‎compiler-rt/lib/fuzzer/FuzzerDefs.h

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
#include <cstring>
1919
#include <string>
2020
#include <vector>
21+
#include <set>
22+
#include <memory>
2123

2224
// Platform detection.
2325
#ifdef __linux__
@@ -102,8 +104,23 @@ struct ExternalFunctions;
102104
// Global interface to functions that may or may not be available.
103105
extern ExternalFunctions *EF;
104106

105-
typedef std::vector<uint8_t> Unit;
106-
typedef std::vector<Unit> UnitVector;
107+
// We are using a custom allocator to give a different symbol name to STL
108+
// containers in order to avoid ODR violations.
109+
template<typename T>
110+
class fuzzer_allocator: public std::allocator<T> {
111+
public:
112+
template<class Other>
113+
struct rebind { typedef fuzzer_allocator<Other> other; };
114+
};
115+
116+
template<typename T>
117+
using Vector = std::vector<T, fuzzer_allocator<T>>;
118+
119+
template<typename T>
120+
using Set = std::set<T, std::less<T>, fuzzer_allocator<T>>;
121+
122+
typedef Vector<uint8_t> Unit;
123+
typedef Vector<Unit> UnitVector;
107124
typedef int (*UserCallback)(const uint8_t *Data, size_t Size);
108125

109126
int FuzzerDriver(int *argc, char ***argv, UserCallback Callback);

‎compiler-rt/lib/fuzzer/FuzzerDictionary.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ class Dictionary {
120120
bool ParseOneDictionaryEntry(const std::string &Str, Unit *U);
121121
// Parses the dictionary file, fills Units, returns true iff all lines
122122
// were parsed succesfully.
123-
bool ParseDictionaryFile(const std::string &Text, std::vector<Unit> *Units);
123+
bool ParseDictionaryFile(const std::string &Text, Vector<Unit> *Units);
124124

125125
} // namespace fuzzer
126126

‎compiler-rt/lib/fuzzer/FuzzerDriver.cpp

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ static const FlagDescription FlagDescriptions [] {
7474
static const size_t kNumFlags =
7575
sizeof(FlagDescriptions) / sizeof(FlagDescriptions[0]);
7676

77-
static std::vector<std::string> *Inputs;
77+
static Vector<std::string> *Inputs;
7878
static std::string *ProgName;
7979

8080
static void PrintHelp() {
@@ -175,7 +175,7 @@ static bool ParseOneFlag(const char *Param) {
175175
}
176176

177177
// We don't use any library to minimize dependencies.
178-
static void ParseFlags(const std::vector<std::string> &Args) {
178+
static void ParseFlags(const Vector<std::string> &Args) {
179179
for (size_t F = 0; F < kNumFlags; F++) {
180180
if (FlagDescriptions[F].IntFlag)
181181
*FlagDescriptions[F].IntFlag = FlagDescriptions[F].Default;
@@ -185,7 +185,7 @@ static void ParseFlags(const std::vector<std::string> &Args) {
185185
if (FlagDescriptions[F].StrFlag)
186186
*FlagDescriptions[F].StrFlag = nullptr;
187187
}
188-
Inputs = new std::vector<std::string>;
188+
Inputs = new Vector<std::string>;
189189
for (size_t A = 1; A < Args.size(); A++) {
190190
if (ParseOneFlag(Args[A].c_str())) {
191191
if (Flags.ignore_remaining_args)
@@ -225,7 +225,7 @@ static void WorkerThread(const std::string &Cmd, std::atomic<unsigned> *Counter,
225225
}
226226
}
227227

228-
std::string CloneArgsWithoutX(const std::vector<std::string> &Args,
228+
std::string CloneArgsWithoutX(const Vector<std::string> &Args,
229229
const char *X1, const char *X2) {
230230
std::string Cmd;
231231
for (auto &S : Args) {
@@ -236,12 +236,12 @@ std::string CloneArgsWithoutX(const std::vector<std::string> &Args,
236236
return Cmd;
237237
}
238238

239-
static int RunInMultipleProcesses(const std::vector<std::string> &Args,
239+
static int RunInMultipleProcesses(const Vector<std::string> &Args,
240240
unsigned NumWorkers, unsigned NumJobs) {
241241
std::atomic<unsigned> Counter(0);
242242
std::atomic<bool> HasErrors(false);
243243
std::string Cmd = CloneArgsWithoutX(Args, "jobs", "workers");
244-
std::vector<std::thread> V;
244+
Vector<std::thread> V;
245245
std::thread Pulse(PulseThread);
246246
Pulse.detach();
247247
for (unsigned i = 0; i < NumWorkers; i++)
@@ -294,7 +294,7 @@ static std::string GetDedupTokenFromFile(const std::string &Path) {
294294
return S.substr(Beg, End - Beg);
295295
}
296296

297-
int CleanseCrashInput(const std::vector<std::string> &Args,
297+
int CleanseCrashInput(const Vector<std::string> &Args,
298298
const FuzzingOptions &Options) {
299299
if (Inputs->size() != 1 || !Flags.exact_artifact_path) {
300300
Printf("ERROR: -cleanse_crash should be given one input file and"
@@ -322,7 +322,7 @@ int CleanseCrashInput(const std::vector<std::string> &Args,
322322
auto U = FileToVector(CurrentFilePath);
323323
size_t Size = U.size();
324324

325-
const std::vector<uint8_t> ReplacementBytes = {' ', 0xff};
325+
const Vector<uint8_t> ReplacementBytes = {' ', 0xff};
326326
for (int NumAttempts = 0; NumAttempts < 5; NumAttempts++) {
327327
bool Changed = false;
328328
for (size_t Idx = 0; Idx < Size; Idx++) {
@@ -354,7 +354,7 @@ int CleanseCrashInput(const std::vector<std::string> &Args,
354354
return 0;
355355
}
356356

357-
int MinimizeCrashInput(const std::vector<std::string> &Args,
357+
int MinimizeCrashInput(const Vector<std::string> &Args,
358358
const FuzzingOptions &Options) {
359359
if (Inputs->size() != 1) {
360360
Printf("ERROR: -minimize_crash should be given one input file\n");
@@ -456,17 +456,17 @@ int MinimizeCrashInputInternalStep(Fuzzer *F, InputCorpus *Corpus) {
456456
return 0;
457457
}
458458

459-
int AnalyzeDictionary(Fuzzer *F, const std::vector<Unit>& Dict,
459+
int AnalyzeDictionary(Fuzzer *F, const Vector<Unit>& Dict,
460460
UnitVector& Corpus) {
461461
Printf("Started dictionary minimization (up to %d tests)\n",
462462
Dict.size() * Corpus.size() * 2);
463463

464464
// Scores and usage count for each dictionary unit.
465-
std::vector<int> Scores(Dict.size());
466-
std::vector<int> Usages(Dict.size());
465+
Vector<int> Scores(Dict.size());
466+
Vector<int> Usages(Dict.size());
467467

468-
std::vector<size_t> InitialFeatures;
469-
std::vector<size_t> ModifiedFeatures;
468+
Vector<size_t> InitialFeatures;
469+
Vector<size_t> ModifiedFeatures;
470470
for (auto &C : Corpus) {
471471
// Get coverage for the testcase without modifications.
472472
F->ExecuteCallback(C.data(), C.size());
@@ -477,7 +477,7 @@ int AnalyzeDictionary(Fuzzer *F, const std::vector<Unit>& Dict,
477477
});
478478

479479
for (size_t i = 0; i < Dict.size(); ++i) {
480-
auto Data = C;
480+
Vector<uint8_t> Data = C;
481481
auto StartPos = std::search(Data.begin(), Data.end(),
482482
Dict[i].begin(), Dict[i].end());
483483
// Skip dictionary unit, if the testcase does not contain it.
@@ -531,7 +531,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
531531
EF = new ExternalFunctions();
532532
if (EF->LLVMFuzzerInitialize)
533533
EF->LLVMFuzzerInitialize(argc, argv);
534-
const std::vector<std::string> Args(*argv, *argv + *argc);
534+
const Vector<std::string> Args(*argv, *argv + *argc);
535535
assert(!Args.empty());
536536
ProgName = new std::string(Args[0]);
537537
if (Argv0 != *ProgName) {
@@ -593,7 +593,7 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
593593
Options.ArtifactPrefix = Flags.artifact_prefix;
594594
if (Flags.exact_artifact_path)
595595
Options.ExactArtifactPath = Flags.exact_artifact_path;
596-
std::vector<Unit> Dictionary;
596+
Vector<Unit> Dictionary;
597597
if (Flags.dict)
598598
if (!ParseDictionaryFile(FileToString(Flags.dict), &Dictionary))
599599
return 1;

‎compiler-rt/lib/fuzzer/FuzzerIO.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,10 +68,10 @@ void WriteToFile(const Unit &U, const std::string &Path) {
6868
fclose(Out);
6969
}
7070

71-
void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V,
71+
void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V,
7272
long *Epoch, size_t MaxSize, bool ExitOnError) {
7373
long E = Epoch ? *Epoch : 0;
74-
std::vector<std::string> Files;
74+
Vector<std::string> Files;
7575
ListFilesInDirRecursive(Path, Epoch, &Files, /*TopDir*/true);
7676
size_t NumLoaded = 0;
7777
for (size_t i = 0; i < Files.size(); i++) {

‎compiler-rt/lib/fuzzer/FuzzerIO.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ void CopyFileToErr(const std::string &Path);
2727

2828
void WriteToFile(const Unit &U, const std::string &Path);
2929

30-
void ReadDirToVectorOfUnits(const char *Path, std::vector<Unit> *V,
30+
void ReadDirToVectorOfUnits(const char *Path, Vector<Unit> *V,
3131
long *Epoch, size_t MaxSize, bool ExitOnError);
3232

3333
// Returns "Dir/FileName" or equivalent for the current OS.
@@ -55,7 +55,7 @@ void RawPrint(const char *Str);
5555
bool IsFile(const std::string &Path);
5656

5757
void ListFilesInDirRecursive(const std::string &Dir, long *Epoch,
58-
std::vector<std::string> *V, bool TopDir);
58+
Vector<std::string> *V, bool TopDir);
5959

6060
char GetSeparator();
6161

‎compiler-rt/lib/fuzzer/FuzzerIOPosix.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ bool IsFile(const std::string &Path) {
3333
}
3434

3535
void ListFilesInDirRecursive(const std::string &Dir, long *Epoch,
36-
std::vector<std::string> *V, bool TopDir) {
36+
Vector<std::string> *V, bool TopDir) {
3737
auto E = GetEpoch(Dir);
3838
if (Epoch)
3939
if (E && *Epoch >= E) return;

‎compiler-rt/lib/fuzzer/FuzzerIOWindows.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ bool IsFile(const std::string &Path) {
7373
}
7474

7575
void ListFilesInDirRecursive(const std::string &Dir, long *Epoch,
76-
std::vector<std::string> *V, bool TopDir) {
76+
Vector<std::string> *V, bool TopDir) {
7777
auto E = GetEpoch(Dir);
7878
if (Epoch)
7979
if (E && *Epoch >= E) return;

‎compiler-rt/lib/fuzzer/FuzzerInternal.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -69,9 +69,9 @@ class Fuzzer {
6969
InputInfo *II = nullptr);
7070

7171
// Merge Corpora[1:] into Corpora[0].
72-
void Merge(const std::vector<std::string> &Corpora);
73-
void CrashResistantMerge(const std::vector<std::string> &Args,
74-
const std::vector<std::string> &Corpora,
72+
void Merge(const Vector<std::string> &Corpora);
73+
void CrashResistantMerge(const Vector<std::string> &Args,
74+
const Vector<std::string> &Corpora,
7575
const char *CoverageSummaryInputPathOrNull,
7676
const char *CoverageSummaryOutputPathOrNull);
7777
void CrashResistantMergeInternalStep(const std::string &ControlFilePath);
@@ -139,7 +139,7 @@ class Fuzzer {
139139
size_t MaxMutationLen = 0;
140140
size_t TmpMaxMutationLen = 0;
141141

142-
std::vector<uint32_t> UniqFeatureSetTmp;
142+
Vector<uint32_t> UniqFeatureSetTmp;
143143

144144
// Need to know our own thread.
145145
static thread_local bool IsMyThread;

‎compiler-rt/lib/fuzzer/FuzzerLoop.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -327,7 +327,7 @@ void Fuzzer::SetMaxMutationLen(size_t MaxMutationLen) {
327327

328328
void Fuzzer::CheckExitOnSrcPosOrItem() {
329329
if (!Options.ExitOnSrcPos.empty()) {
330-
static auto *PCsSet = new std::set<uintptr_t>;
330+
static auto *PCsSet = new Set<uintptr_t>;
331331
auto HandlePC = [&](uintptr_t PC) {
332332
if (!PCsSet->insert(PC).second) return;
333333
std::string Descr = DescribePC("%F %L", PC + 1);
@@ -350,7 +350,7 @@ void Fuzzer::CheckExitOnSrcPosOrItem() {
350350

351351
void Fuzzer::RereadOutputCorpus(size_t MaxSize) {
352352
if (Options.OutputCorpus.empty() || !Options.ReloadIntervalSec) return;
353-
std::vector<Unit> AdditionalCorpus;
353+
Vector<Unit> AdditionalCorpus;
354354
ReadDirToVectorOfUnits(Options.OutputCorpus.c_str(), &AdditionalCorpus,
355355
&EpochOfLastReadOfOutputCorpus, MaxSize,
356356
/*ExitOnError*/ false);

‎compiler-rt/lib/fuzzer/FuzzerMerge.cpp

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ bool Merger::Parse(std::istream &IS, bool ParseCoverage) {
7474
size_t ExpectedStartMarker = 0;
7575
const size_t kInvalidStartMarker = -1;
7676
size_t LastSeenStartMarker = kInvalidStartMarker;
77-
std::vector<uint32_t> TmpFeatures;
77+
Vector<uint32_t> TmpFeatures;
7878
while (std::getline(IS, Line, '\n')) {
7979
std::istringstream ISS1(Line);
8080
std::string Marker;
@@ -122,11 +122,11 @@ size_t Merger::ApproximateMemoryConsumption() const {
122122

123123
// Decides which files need to be merged (add thost to NewFiles).
124124
// Returns the number of new features added.
125-
size_t Merger::Merge(const std::set<uint32_t> &InitialFeatures,
126-
std::vector<std::string> *NewFiles) {
125+
size_t Merger::Merge(const Set<uint32_t> &InitialFeatures,
126+
Vector<std::string> *NewFiles) {
127127
NewFiles->clear();
128128
assert(NumFilesInFirstCorpus <= Files.size());
129-
std::set<uint32_t> AllFeatures(InitialFeatures);
129+
Set<uint32_t> AllFeatures(InitialFeatures);
130130

131131
// What features are in the initial corpus?
132132
for (size_t i = 0; i < NumFilesInFirstCorpus; i++) {
@@ -138,7 +138,7 @@ size_t Merger::Merge(const std::set<uint32_t> &InitialFeatures,
138138
// Remove all features that we already know from all other inputs.
139139
for (size_t i = NumFilesInFirstCorpus; i < Files.size(); i++) {
140140
auto &Cur = Files[i].Features;
141-
std::vector<uint32_t> Tmp;
141+
Vector<uint32_t> Tmp;
142142
std::set_difference(Cur.begin(), Cur.end(), AllFeatures.begin(),
143143
AllFeatures.end(), std::inserter(Tmp, Tmp.begin()));
144144
Cur.swap(Tmp);
@@ -178,16 +178,16 @@ void Merger::PrintSummary(std::ostream &OS) {
178178
}
179179
}
180180

181-
std::set<uint32_t> Merger::AllFeatures() const {
182-
std::set<uint32_t> S;
181+
Set<uint32_t> Merger::AllFeatures() const {
182+
Set<uint32_t> S;
183183
for (auto &File : Files)
184184
S.insert(File.Features.begin(), File.Features.end());
185185
return S;
186186
}
187187

188-
std::set<uint32_t> Merger::ParseSummary(std::istream &IS) {
188+
Set<uint32_t> Merger::ParseSummary(std::istream &IS) {
189189
std::string Line, Tmp;
190-
std::set<uint32_t> Res;
190+
Set<uint32_t> Res;
191191
while (std::getline(IS, Line, '\n')) {
192192
size_t N;
193193
std::istringstream ISS1(Line);
@@ -235,7 +235,7 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) {
235235
TPC.ResetMaps();
236236
ExecuteCallback(U.data(), U.size());
237237
// Collect coverage.
238-
std::set<size_t> Features;
238+
Set<size_t> Features;
239239
TPC.CollectFeatures([&](size_t Feature) -> bool {
240240
Features.insert(Feature);
241241
return true;
@@ -252,15 +252,15 @@ void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) {
252252
}
253253

254254
// Outer process. Does not call the target code and thus sohuld not fail.
255-
void Fuzzer::CrashResistantMerge(const std::vector<std::string> &Args,
256-
const std::vector<std::string> &Corpora,
255+
void Fuzzer::CrashResistantMerge(const Vector<std::string> &Args,
256+
const Vector<std::string> &Corpora,
257257
const char *CoverageSummaryInputPathOrNull,
258258
const char *CoverageSummaryOutputPathOrNull) {
259259
if (Corpora.size() <= 1) {
260260
Printf("Merge requires two or more corpus dirs\n");
261261
return;
262262
}
263-
std::vector<std::string> AllFiles;
263+
Vector<std::string> AllFiles;
264264
ListFilesInDirRecursive(Corpora[0], nullptr, &AllFiles, /*TopDir*/true);
265265
size_t NumFilesInFirstCorpus = AllFiles.size();
266266
for (size_t i = 1; i < Corpora.size(); i++)
@@ -318,8 +318,8 @@ void Fuzzer::CrashResistantMerge(const std::vector<std::string> &Args,
318318
std::ofstream SummaryOut(CoverageSummaryOutputPathOrNull);
319319
M.PrintSummary(SummaryOut);
320320
}
321-
std::vector<std::string> NewFiles;
322-
std::set<uint32_t> InitialFeatures;
321+
Vector<std::string> NewFiles;
322+
Set<uint32_t> InitialFeatures;
323323
if (CoverageSummaryInputPathOrNull) {
324324
std::ifstream SummaryIn(CoverageSummaryInputPathOrNull);
325325
InitialFeatures = M.ParseSummary(SummaryIn);

0 commit comments

Comments
 (0)