Skip to content

Commit 535fb60

Browse files
vmaksimovmaksimo
committed
Update --parallel-exec option to take number of maximum worker threads to use
1 parent 782277c commit 535fb60

File tree

3 files changed

+74
-34
lines changed

3 files changed

+74
-34
lines changed

‎llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,14 @@
66
; RUN: echo "%t2.tgt" >> %t.list
77
; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
88
; RUN: FileCheck < %t.res %s
9-
; RUN: llvm-foreach --parallel-exec --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
9+
; RUN: llvm-foreach --parallel-exec=2 --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
1010
; RUN: FileCheck < %t.res %s
1111
; CHECK-DAG: [[FIRST:.+1.tgt]]
1212
; CHECK-DAG: [[SECOND:.+2.tgt]]
1313
;
1414
; RUN: llvm-foreach --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- cp "{}" %t
1515
; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST
16-
; RUN: llvm-foreach --parallel-exec --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- cp "{}" %t
16+
; RUN: llvm-foreach --parallel-exec=2 --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- cp "{}" %t
1717
; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST
1818
; CHECK-LIST-DAG: [[FIRST:.+\.out]]
1919
; CHECK-LIST-DAG: [[SECOND:.+\.out]]
@@ -26,7 +26,18 @@
2626
; RUN: echo "%t4.tgt" >> %t1.list
2727
; RUN: llvm-foreach --in-replace="{}" --in-replace="inrep" --in-file-list=%t.list --in-file-list=%t1.list --out-increment="%t_out.prj" -- echo -first-part-of-arg={}.out -first-part-of-arg=inrep.out -another-arg=%t_out.prj > %t1.res
2828
; RUN: FileCheck < %t1.res %s --check-prefix=CHECK-DOUBLE-LISTS
29-
; RUN: llvm-foreach --parallel-exec --in-replace="{}" --in-replace="inrep" --in-file-list=%t.list --in-file-list=%t1.list --out-increment="%t_out.prj" -- echo -first-part-of-arg={}.out -first-part-of-arg=inrep.out -another-arg=%t_out.prj > %t1.res
29+
; RUN: llvm-foreach --parallel-exec=2 --in-replace="{}" --in-replace="inrep" --in-file-list=%t.list --in-file-list=%t1.list --out-increment="%t_out.prj" -- echo -first-part-of-arg={}.out -first-part-of-arg=inrep.out -another-arg=%t_out.prj > %t1.res
3030
; RUN: FileCheck < %t1.res %s --check-prefix=CHECK-DOUBLE-LISTS
3131
; CHECK-DOUBLE-LISTS-DAG: -first-part-of-arg=[[FIRST:.+1.tgt.out]] -first-part-of-arg=[[THIRD:.+3.tgt.out]] -another-arg={{.+}}_out.prj
3232
; CHECK-DOUBLE-LISTS-DAG: -first-part-of-arg=[[SECOND:.+2.tgt.out]] -first-part-of-arg=[[FOURTH:.+4.tgt.out]] -another-arg={{.+}}_out.prj_1
33+
34+
; RUN: echo "%t1.tgt" > %t2.list
35+
; RUN: echo "%t2.tgt" >> %t2.list
36+
; RUN: echo "%t3.tgt" >> %t2.list
37+
; RUN: echo "%t4.tgt" >> %t2.list
38+
; RUN: llvm-foreach --parallel-exec=2 --in-replace="{}" --in-file-list=%t2.list -- echo "{}" > %t2.res
39+
; RUN: FileCheck < %t2.res %s --check-prefix=CHECK-PARALLEL-EXEC
40+
; CHECK-PARALLEL-EXEC-DAG: [[FIRST:.+1.tgt]]
41+
; CHECK-PARALLEL-EXEC-DAG: [[SECOND:.+2.tgt]]
42+
; CHECK-PARALLEL-EXEC-DAG: [[THIRD:.+3.tgt]]
43+
; CHECK-PARALLEL-EXEC-DAG: [[FOURTH:.+4.tgt]]

‎llvm/test/tools/llvm-foreach/llvm-foreach-win.ll

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,14 @@
66
; RUN: echo "%t2.tgt" >> %t.list
77
; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
88
; RUN: FileCheck < %t.res %s
9-
; RUN: llvm-foreach --parallel-exec --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
9+
; RUN: llvm-foreach --parallel-exec=2 --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
1010
; RUN: FileCheck < %t.res %s
1111
; CHECK-DAG: [[FIRST:.+1.tgt]]
1212
; CHECK-DAG: [[SECOND:.+2.tgt]]
1313
;
1414
; RUN: llvm-foreach --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- xcopy /y "{}" %t
1515
; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST
16-
; RUN: llvm-foreach --parallel-exec --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- xcopy /y "{}" %t
16+
; RUN: llvm-foreach --parallel-exec=2 --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- xcopy /y "{}" %t
1717
; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST
1818
; CHECK-LIST-DAG: [[FIRST:.+\.out]]
1919
; CHECK-LIST-DAG: [[SECOND:.+\.out]]
@@ -26,7 +26,18 @@
2626
; RUN: echo "%t4.tgt" >> %t1.list
2727
; RUN: llvm-foreach --in-replace="{}" --in-replace="inrep" --in-file-list=%t.list --in-file-list=%t1.list --out-increment=%t_out.prj -- echo -first-part-of-arg={}.out -first-part-of-arg=inrep.out -another-arg=%t_out.prj > %t1.res
2828
; RUN: FileCheck < %t1.res %s --check-prefix=CHECK-DOUBLE-LISTS
29-
; RUN: llvm-foreach --parallel-exec --in-replace="{}" --in-replace="inrep" --in-file-list=%t.list --in-file-list=%t1.list --out-increment=%t_out.prj -- echo -first-part-of-arg={}.out -first-part-of-arg=inrep.out -another-arg=%t_out.prj > %t1.res
29+
; RUN: llvm-foreach --parallel-exec=2 --in-replace="{}" --in-replace="inrep" --in-file-list=%t.list --in-file-list=%t1.list --out-increment=%t_out.prj -- echo -first-part-of-arg={}.out -first-part-of-arg=inrep.out -another-arg=%t_out.prj > %t1.res
3030
; RUN: FileCheck < %t1.res %s --check-prefix=CHECK-DOUBLE-LISTS
3131
; CHECK-DOUBLE-LISTS-DAG: -first-part-of-arg=[[FIRST:.+1.tgt.out]] -first-part-of-arg=[[THIRD:.+3.tgt.out]] -another-arg={{.+}}_out.prj
3232
; CHECK-DOUBLE-LISTS-DAG: -first-part-of-arg=[[SECOND:.+2.tgt.out]] -first-part-of-arg=[[FOURTH:.+4.tgt.out]] -another-arg={{.+}}_out.prj_1
33+
34+
; RUN: echo "%t1.tgt" > %t2.list
35+
; RUN: echo "%t2.tgt" >> %t2.list
36+
; RUN: echo "%t3.tgt" >> %t2.list
37+
; RUN: echo "%t4.tgt" >> %t2.list
38+
; RUN: llvm-foreach --parallel-exec=2 --in-replace="{}" --in-file-list=%t2.list -- echo "{}" > %t2.res
39+
; RUN: FileCheck < %t2.res %s --check-prefix=CHECK-PARALLEL-EXEC
40+
; CHECK-PARALLEL-EXEC-DAG: [[FIRST:.+1.tgt]]
41+
; CHECK-PARALLEL-EXEC-DAG: [[SECOND:.+2.tgt]]
42+
; CHECK-PARALLEL-EXEC-DAG: [[THIRD:.+3.tgt]]
43+
; CHECK-PARALLEL-EXEC-DAG: [[FOURTH:.+4.tgt]]

‎llvm/tools/llvm-foreach/llvm-foreach.cpp

Lines changed: 46 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -70,10 +70,13 @@ static cl::opt<std::string> OutIncrement{
7070
"pass."),
7171
cl::init(""), cl::value_desc("R")};
7272

73-
static cl::opt<bool> ExecuteInParallel{
73+
static cl::opt<unsigned int> ExecuteInParallel{
7474
"parallel-exec",
75-
cl::desc("Enable launching input commands in parallel mode"),
76-
cl::init(false)};
75+
cl::Optional,
76+
cl::init(1),
77+
cl::desc("Specify the number of threads for launching input commands in "
78+
"parallel mode"),
79+
};
7780

7881
static void error(const Twine &Msg) {
7982
errs() << "llvm-foreach: " << Msg << '\n';
@@ -85,6 +88,31 @@ static void error(std::error_code EC, const Twine &Prefix) {
8588
error(Prefix + ": " + EC.message());
8689
}
8790

91+
// With WaitUntilTerminates=false this function just goes through the all
92+
// submitted jobs to check if one of them has finished.
93+
// TODO: give a proper naming as this function doesn't wait for jobs with
94+
// WaitUntilTerminates=false
95+
void waitJobsToBeFinished(std::list<sys::ProcessInfo> &JobsSubmitted, int &Res,
96+
bool WaitUntilTerminates = true) {
97+
std::string ErrMsg;
98+
auto It = JobsSubmitted.begin();
99+
while (It != JobsSubmitted.end()) {
100+
sys::ProcessInfo WaitResult =
101+
sys::Wait(*It, 0, /*WaitUntilTerminates*/ WaitUntilTerminates, &ErrMsg);
102+
103+
// Check if the job has finished (PID will be 0 if it's not).
104+
if (!WaitUntilTerminates && !WaitResult.Pid) {
105+
It++;
106+
continue;
107+
}
108+
if (WaitResult.ReturnCode != 0) {
109+
errs() << "llvm-foreach: " << ErrMsg << '\n';
110+
Res = WaitResult.ReturnCode;
111+
}
112+
It = JobsSubmitted.erase(It);
113+
}
114+
}
115+
88116
int main(int argc, char **argv) {
89117
cl::ParseCommandLineOptions(
90118
argc, argv,
@@ -166,6 +194,13 @@ int main(int argc, char **argv) {
166194
PrevNumOfLines = FileLists[i].size();
167195
}
168196

197+
if (!ExecuteInParallel)
198+
error("Number of parallel threads should be a positive integer");
199+
if (ExecuteInParallel > 16) {
200+
ExecuteInParallel = 16;
201+
errs() << "llvm-foreach: adjusted number of threads to 16 (max available)";
202+
}
203+
169204
std::error_code EC;
170205
raw_fd_ostream OS{OutputFileList, EC, sys::fs::OpenFlags::OF_None};
171206
if (!OutputFileList.empty())
@@ -176,7 +211,7 @@ int main(int argc, char **argv) {
176211
std::string IncOutArg;
177212
std::vector<std::string> ResInArgs(InReplaceArgs.size());
178213
std::string ResFileList = "";
179-
std::list<sys::ProcessInfo> CommandsStarted;
214+
std::list<sys::ProcessInfo> JobsSubmitted;
180215
for (size_t j = 0; j != FileLists[0].size(); ++j) {
181216
for (size_t i = 0; i < InReplaceArgs.size(); ++i) {
182217
ArgumentReplace CurReplace = InReplaceArgs[i];
@@ -228,34 +263,17 @@ int main(int argc, char **argv) {
228263
Args[OutIncrementArg.ArgNum] = IncOutArg;
229264
}
230265

231-
std::string ErrMsg;
232-
if (ExecuteInParallel) {
233-
CommandsStarted.emplace_back(sys::ExecuteNoWait(
234-
Prog, Args, /*Env=*/None, /*Redirects=*/None, /*MemoryLimit=*/0));
235-
continue;
236-
}
266+
// Do not start execution of a new job until previous one(s) are finished,
267+
// if the maximum number of parallel workers is reached.
268+
while (JobsSubmitted.size() == ExecuteInParallel)
269+
waitJobsToBeFinished(JobsSubmitted, Res, /*WaitUntilTerminates*/ false);
237270

238-
int Result =
239-
sys::ExecuteAndWait(Prog, Args, /*Env=*/None, /*Redirects=*/None,
240-
/*SecondsToWait=*/0, /*MemoryLimit=*/0, &ErrMsg);
241-
if (Result != 0) {
242-
errs() << "llvm-foreach: " << ErrMsg << '\n';
243-
Res = Result;
244-
}
271+
JobsSubmitted.emplace_back(sys::ExecuteNoWait(
272+
Prog, Args, /*Env=*/None, /*Redirects=*/None, /*MemoryLimit=*/0));
245273
}
246274

247275
// Wait for all commands to be executed.
248-
std::string ErrMsg;
249-
auto It = CommandsStarted.begin();
250-
while (It != CommandsStarted.end()) {
251-
sys::ProcessInfo WaitResult =
252-
sys::Wait(*It, 0, /*WaitUntilTerminates*/ true, &ErrMsg);
253-
if (WaitResult.ReturnCode != 0) {
254-
errs() << "llvm-foreach: " << ErrMsg << '\n';
255-
Res = WaitResult.ReturnCode;
256-
}
257-
It = CommandsStarted.erase(It);
258-
}
276+
waitJobsToBeFinished(JobsSubmitted, Res, /*WaitUntilTerminates*/ true);
259277

260278
if (!OutputFileList.empty()) {
261279
OS.close();

0 commit comments

Comments
 (0)