From 782277caf14555c1e74c76b9b2d2a8c72a2401ff Mon Sep 17 00:00:00 2001
From: Viktoria Maksimova <viktoria.maksimova@intel.com>
Date: Tue, 17 Aug 2021 22:43:36 +0300
Subject: [PATCH 01/11] [SYCL] Enable parallel execution of llvm-foreach
 commands under an option

This improvement should speed up toolchain execution by launching underlying
llvm-foreach commands in parallel.
The feature can be enabled by passing '--parallel-exec' to the command line of
llvm-foreach.
---
 .../tools/llvm-foreach/llvm-foreach-lin.ll    | 18 ++++++++-----
 .../tools/llvm-foreach/llvm-foreach-win.ll    | 18 ++++++++-----
 llvm/tools/llvm-foreach/llvm-foreach.cpp      | 27 ++++++++++++++++++-
 3 files changed, 50 insertions(+), 13 deletions(-)

diff --git a/llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll b/llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll
index 2fa674bf1d8e3..07303f034952a 100644
--- a/llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll
+++ b/llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll
@@ -6,13 +6,17 @@
 ; RUN: echo "%t2.tgt" >> %t.list
 ; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
 ; RUN: FileCheck < %t.res %s
-; CHECK: [[FIRST:.+1.tgt]]
-; CHECK: [[SECOND:.+2.tgt]]
+; RUN: llvm-foreach --parallel-exec --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
+; RUN: FileCheck < %t.res %s
+; CHECK-DAG: [[FIRST:.+1.tgt]]
+; CHECK-DAG: [[SECOND:.+2.tgt]]
 ;
 ; RUN: llvm-foreach --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- cp "{}" %t
 ; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST
-; CHECK-LIST: [[FIRST:.+\.out]]
-; CHECK-LIST: [[SECOND:.+\.out]]
+; RUN: llvm-foreach --parallel-exec --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- cp "{}" %t
+; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST
+; CHECK-LIST-DAG: [[FIRST:.+\.out]]
+; CHECK-LIST-DAG: [[SECOND:.+\.out]]
 ; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.out.list -- FileCheck --input-file="{}" %s --check-prefix=CHECK-CONTENT
 ; CHECK-CONTENT: Content of
 
@@ -22,5 +26,7 @@
 ; RUN: echo "%t4.tgt" >> %t1.list
 ; RUN: llvm-foreach --in-replace="{}" --in-replace="inrep" --in-file-list=%t.list --in-file-list=%t1.list --out-increment="%t_out.prj" -- echo -first-part-of-arg={}.out -first-part-of-arg=inrep.out -another-arg=%t_out.prj > %t1.res
 ; RUN: FileCheck < %t1.res %s --check-prefix=CHECK-DOUBLE-LISTS
-; CHECK-DOUBLE-LISTS: -first-part-of-arg=[[FIRST:.+1.tgt.out]] -first-part-of-arg=[[THIRD:.+3.tgt.out]] -another-arg={{.+}}_out.prj
-; CHECK-DOUBLE-LISTS: -first-part-of-arg=[[SECOND:.+2.tgt.out]] -first-part-of-arg=[[FOURTH:.+4.tgt.out]] -another-arg={{.+}}_out.prj_1
+; RUN: llvm-foreach --parallel-exec --in-replace="{}" --in-replace="inrep" --in-file-list=%t.list --in-file-list=%t1.list --out-increment="%t_out.prj" -- echo -first-part-of-arg={}.out -first-part-of-arg=inrep.out -another-arg=%t_out.prj > %t1.res
+; RUN: FileCheck < %t1.res %s --check-prefix=CHECK-DOUBLE-LISTS
+; CHECK-DOUBLE-LISTS-DAG: -first-part-of-arg=[[FIRST:.+1.tgt.out]] -first-part-of-arg=[[THIRD:.+3.tgt.out]] -another-arg={{.+}}_out.prj
+; CHECK-DOUBLE-LISTS-DAG: -first-part-of-arg=[[SECOND:.+2.tgt.out]] -first-part-of-arg=[[FOURTH:.+4.tgt.out]] -another-arg={{.+}}_out.prj_1
diff --git a/llvm/test/tools/llvm-foreach/llvm-foreach-win.ll b/llvm/test/tools/llvm-foreach/llvm-foreach-win.ll
index 9053e13c74a64..2627a3122f35d 100644
--- a/llvm/test/tools/llvm-foreach/llvm-foreach-win.ll
+++ b/llvm/test/tools/llvm-foreach/llvm-foreach-win.ll
@@ -6,13 +6,17 @@
 ; RUN: echo "%t2.tgt" >> %t.list
 ; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
 ; RUN: FileCheck < %t.res %s
-; CHECK: [[FIRST:.+1.tgt]]
-; CHECK: [[SECOND:.+2.tgt]]
+; RUN: llvm-foreach --parallel-exec --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
+; RUN: FileCheck < %t.res %s
+; CHECK-DAG: [[FIRST:.+1.tgt]]
+; CHECK-DAG: [[SECOND:.+2.tgt]]
 ;
 ; RUN: llvm-foreach --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- xcopy /y "{}" %t
 ; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST
-; CHECK-LIST: [[FIRST:.+\.out]]
-; CHECK-LIST: [[SECOND:.+\.out]]
+; RUN: llvm-foreach --parallel-exec --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- xcopy /y "{}" %t
+; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST
+; CHECK-LIST-DAG: [[FIRST:.+\.out]]
+; CHECK-LIST-DAG: [[SECOND:.+\.out]]
 ; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.out.list -- FileCheck --input-file="{}" %s --check-prefix=CHECK-CONTENT
 ; CHECK-CONTENT: Content of
 
@@ -22,5 +26,7 @@
 ; RUN: echo "%t4.tgt" >> %t1.list
 ; RUN: llvm-foreach --in-replace="{}" --in-replace="inrep" --in-file-list=%t.list --in-file-list=%t1.list --out-increment=%t_out.prj -- echo -first-part-of-arg={}.out -first-part-of-arg=inrep.out -another-arg=%t_out.prj > %t1.res
 ; RUN: FileCheck < %t1.res %s --check-prefix=CHECK-DOUBLE-LISTS
-; CHECK-DOUBLE-LISTS: -first-part-of-arg=[[FIRST:.+1.tgt.out]] -first-part-of-arg=[[THIRD:.+3.tgt.out]] -another-arg={{.+}}_out.prj
-; CHECK-DOUBLE-LISTS: -first-part-of-arg=[[SECOND:.+2.tgt.out]] -first-part-of-arg=[[FOURTH:.+4.tgt.out]] -another-arg={{.+}}_out.prj_1
+; RUN: llvm-foreach --parallel-exec --in-replace="{}" --in-replace="inrep" --in-file-list=%t.list --in-file-list=%t1.list --out-increment=%t_out.prj -- echo -first-part-of-arg={}.out -first-part-of-arg=inrep.out -another-arg=%t_out.prj > %t1.res
+; RUN: FileCheck < %t1.res %s --check-prefix=CHECK-DOUBLE-LISTS
+; CHECK-DOUBLE-LISTS-DAG: -first-part-of-arg=[[FIRST:.+1.tgt.out]] -first-part-of-arg=[[THIRD:.+3.tgt.out]] -another-arg={{.+}}_out.prj
+; CHECK-DOUBLE-LISTS-DAG: -first-part-of-arg=[[SECOND:.+2.tgt.out]] -first-part-of-arg=[[FOURTH:.+4.tgt.out]] -another-arg={{.+}}_out.prj_1
diff --git a/llvm/tools/llvm-foreach/llvm-foreach.cpp b/llvm/tools/llvm-foreach/llvm-foreach.cpp
index 9a7f1a6031860..1ca858a5a98aa 100644
--- a/llvm/tools/llvm-foreach/llvm-foreach.cpp
+++ b/llvm/tools/llvm-foreach/llvm-foreach.cpp
@@ -19,6 +19,7 @@
 #include "llvm/Support/Program.h"
 #include "llvm/Support/SystemUtils.h"
 
+#include <list>
 #include <vector>
 
 using namespace llvm;
@@ -69,6 +70,11 @@ static cl::opt<std::string> OutIncrement{
         "pass."),
     cl::init(""), cl::value_desc("R")};
 
+static cl::opt<bool> ExecuteInParallel{
+    "parallel-exec",
+    cl::desc("Enable launching input commands in parallel mode"),
+    cl::init(false)};
+
 static void error(const Twine &Msg) {
   errs() << "llvm-foreach: " << Msg << '\n';
   exit(1);
@@ -170,6 +176,7 @@ int main(int argc, char **argv) {
   std::string IncOutArg;
   std::vector<std::string> ResInArgs(InReplaceArgs.size());
   std::string ResFileList = "";
+  std::list<sys::ProcessInfo> CommandsStarted;
   for (size_t j = 0; j != FileLists[0].size(); ++j) {
     for (size_t i = 0; i < InReplaceArgs.size(); ++i) {
       ArgumentReplace CurReplace = InReplaceArgs[i];
@@ -222,7 +229,12 @@ int main(int argc, char **argv) {
     }
 
     std::string ErrMsg;
-    // TODO: Add possibility to execute commands in parallel.
+    if (ExecuteInParallel) {
+      CommandsStarted.emplace_back(sys::ExecuteNoWait(
+          Prog, Args, /*Env=*/None, /*Redirects=*/None, /*MemoryLimit=*/0));
+      continue;
+    }
+
     int Result =
         sys::ExecuteAndWait(Prog, Args, /*Env=*/None, /*Redirects=*/None,
                             /*SecondsToWait=*/0, /*MemoryLimit=*/0, &ErrMsg);
@@ -232,6 +244,19 @@ int main(int argc, char **argv) {
     }
   }
 
+  // Wait for all commands to be executed.
+  std::string ErrMsg;
+  auto It = CommandsStarted.begin();
+  while (It != CommandsStarted.end()) {
+    sys::ProcessInfo WaitResult =
+        sys::Wait(*It, 0, /*WaitUntilTerminates*/ true, &ErrMsg);
+    if (WaitResult.ReturnCode != 0) {
+      errs() << "llvm-foreach: " << ErrMsg << '\n';
+      Res = WaitResult.ReturnCode;
+    }
+    It = CommandsStarted.erase(It);
+  }
+
   if (!OutputFileList.empty()) {
     OS.close();
   }

From 535fb6052b5629c1589837e17a0820e129cca78a Mon Sep 17 00:00:00 2001
From: Viktoria Maksimova <viktoria.maksimova@intel.com>
Date: Thu, 19 Aug 2021 18:05:22 +0300
Subject: [PATCH 02/11] Update --parallel-exec option to take number of maximum
 worker threads to use

---
 .../tools/llvm-foreach/llvm-foreach-lin.ll    | 17 ++++-
 .../tools/llvm-foreach/llvm-foreach-win.ll    | 17 ++++-
 llvm/tools/llvm-foreach/llvm-foreach.cpp      | 74 ++++++++++++-------
 3 files changed, 74 insertions(+), 34 deletions(-)

diff --git a/llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll b/llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll
index 07303f034952a..1608bd945d9b0 100644
--- a/llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll
+++ b/llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll
@@ -6,14 +6,14 @@
 ; RUN: echo "%t2.tgt" >> %t.list
 ; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
 ; RUN: FileCheck < %t.res %s
-; RUN: llvm-foreach --parallel-exec --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
+; RUN: llvm-foreach --parallel-exec=2 --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
 ; RUN: FileCheck < %t.res %s
 ; CHECK-DAG: [[FIRST:.+1.tgt]]
 ; CHECK-DAG: [[SECOND:.+2.tgt]]
 ;
 ; RUN: llvm-foreach --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- cp "{}" %t
 ; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST
-; RUN: llvm-foreach --parallel-exec --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- cp "{}" %t
+; RUN: llvm-foreach --parallel-exec=2 --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- cp "{}" %t
 ; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST
 ; CHECK-LIST-DAG: [[FIRST:.+\.out]]
 ; CHECK-LIST-DAG: [[SECOND:.+\.out]]
@@ -26,7 +26,18 @@
 ; RUN: echo "%t4.tgt" >> %t1.list
 ; RUN: llvm-foreach --in-replace="{}" --in-replace="inrep" --in-file-list=%t.list --in-file-list=%t1.list --out-increment="%t_out.prj" -- echo -first-part-of-arg={}.out -first-part-of-arg=inrep.out -another-arg=%t_out.prj > %t1.res
 ; RUN: FileCheck < %t1.res %s --check-prefix=CHECK-DOUBLE-LISTS
-; RUN: llvm-foreach --parallel-exec --in-replace="{}" --in-replace="inrep" --in-file-list=%t.list --in-file-list=%t1.list --out-increment="%t_out.prj" -- echo -first-part-of-arg={}.out -first-part-of-arg=inrep.out -another-arg=%t_out.prj > %t1.res
+; RUN: llvm-foreach --parallel-exec=2 --in-replace="{}" --in-replace="inrep" --in-file-list=%t.list --in-file-list=%t1.list --out-increment="%t_out.prj" -- echo -first-part-of-arg={}.out -first-part-of-arg=inrep.out -another-arg=%t_out.prj > %t1.res
 ; RUN: FileCheck < %t1.res %s --check-prefix=CHECK-DOUBLE-LISTS
 ; CHECK-DOUBLE-LISTS-DAG: -first-part-of-arg=[[FIRST:.+1.tgt.out]] -first-part-of-arg=[[THIRD:.+3.tgt.out]] -another-arg={{.+}}_out.prj
 ; CHECK-DOUBLE-LISTS-DAG: -first-part-of-arg=[[SECOND:.+2.tgt.out]] -first-part-of-arg=[[FOURTH:.+4.tgt.out]] -another-arg={{.+}}_out.prj_1
+
+; RUN: echo "%t1.tgt" > %t2.list
+; RUN: echo "%t2.tgt" >> %t2.list
+; RUN: echo "%t3.tgt" >> %t2.list
+; RUN: echo "%t4.tgt" >> %t2.list
+; RUN: llvm-foreach --parallel-exec=2 --in-replace="{}" --in-file-list=%t2.list -- echo "{}" > %t2.res
+; RUN: FileCheck < %t2.res %s --check-prefix=CHECK-PARALLEL-EXEC
+; CHECK-PARALLEL-EXEC-DAG: [[FIRST:.+1.tgt]]
+; CHECK-PARALLEL-EXEC-DAG: [[SECOND:.+2.tgt]]
+; CHECK-PARALLEL-EXEC-DAG: [[THIRD:.+3.tgt]]
+; CHECK-PARALLEL-EXEC-DAG: [[FOURTH:.+4.tgt]]
diff --git a/llvm/test/tools/llvm-foreach/llvm-foreach-win.ll b/llvm/test/tools/llvm-foreach/llvm-foreach-win.ll
index 2627a3122f35d..dd6bef14b4d6a 100644
--- a/llvm/test/tools/llvm-foreach/llvm-foreach-win.ll
+++ b/llvm/test/tools/llvm-foreach/llvm-foreach-win.ll
@@ -6,14 +6,14 @@
 ; RUN: echo "%t2.tgt" >> %t.list
 ; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
 ; RUN: FileCheck < %t.res %s
-; RUN: llvm-foreach --parallel-exec --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
+; RUN: llvm-foreach --parallel-exec=2 --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
 ; RUN: FileCheck < %t.res %s
 ; CHECK-DAG: [[FIRST:.+1.tgt]]
 ; CHECK-DAG: [[SECOND:.+2.tgt]]
 ;
 ; RUN: llvm-foreach --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- xcopy /y "{}" %t
 ; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST
-; RUN: llvm-foreach --parallel-exec --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- xcopy /y "{}" %t
+; RUN: llvm-foreach --parallel-exec=2 --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- xcopy /y "{}" %t
 ; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST
 ; CHECK-LIST-DAG: [[FIRST:.+\.out]]
 ; CHECK-LIST-DAG: [[SECOND:.+\.out]]
@@ -26,7 +26,18 @@
 ; RUN: echo "%t4.tgt" >> %t1.list
 ; RUN: llvm-foreach --in-replace="{}" --in-replace="inrep" --in-file-list=%t.list --in-file-list=%t1.list --out-increment=%t_out.prj -- echo -first-part-of-arg={}.out -first-part-of-arg=inrep.out -another-arg=%t_out.prj > %t1.res
 ; RUN: FileCheck < %t1.res %s --check-prefix=CHECK-DOUBLE-LISTS
-; RUN: llvm-foreach --parallel-exec --in-replace="{}" --in-replace="inrep" --in-file-list=%t.list --in-file-list=%t1.list --out-increment=%t_out.prj -- echo -first-part-of-arg={}.out -first-part-of-arg=inrep.out -another-arg=%t_out.prj > %t1.res
+; RUN: llvm-foreach --parallel-exec=2 --in-replace="{}" --in-replace="inrep" --in-file-list=%t.list --in-file-list=%t1.list --out-increment=%t_out.prj -- echo -first-part-of-arg={}.out -first-part-of-arg=inrep.out -another-arg=%t_out.prj > %t1.res
 ; RUN: FileCheck < %t1.res %s --check-prefix=CHECK-DOUBLE-LISTS
 ; CHECK-DOUBLE-LISTS-DAG: -first-part-of-arg=[[FIRST:.+1.tgt.out]] -first-part-of-arg=[[THIRD:.+3.tgt.out]] -another-arg={{.+}}_out.prj
 ; CHECK-DOUBLE-LISTS-DAG: -first-part-of-arg=[[SECOND:.+2.tgt.out]] -first-part-of-arg=[[FOURTH:.+4.tgt.out]] -another-arg={{.+}}_out.prj_1
+
+; RUN: echo "%t1.tgt" > %t2.list
+; RUN: echo "%t2.tgt" >> %t2.list
+; RUN: echo "%t3.tgt" >> %t2.list
+; RUN: echo "%t4.tgt" >> %t2.list
+; RUN: llvm-foreach --parallel-exec=2 --in-replace="{}" --in-file-list=%t2.list -- echo "{}" > %t2.res
+; RUN: FileCheck < %t2.res %s --check-prefix=CHECK-PARALLEL-EXEC
+; CHECK-PARALLEL-EXEC-DAG: [[FIRST:.+1.tgt]]
+; CHECK-PARALLEL-EXEC-DAG: [[SECOND:.+2.tgt]]
+; CHECK-PARALLEL-EXEC-DAG: [[THIRD:.+3.tgt]]
+; CHECK-PARALLEL-EXEC-DAG: [[FOURTH:.+4.tgt]]
diff --git a/llvm/tools/llvm-foreach/llvm-foreach.cpp b/llvm/tools/llvm-foreach/llvm-foreach.cpp
index 1ca858a5a98aa..1728c8511a496 100644
--- a/llvm/tools/llvm-foreach/llvm-foreach.cpp
+++ b/llvm/tools/llvm-foreach/llvm-foreach.cpp
@@ -70,10 +70,13 @@ static cl::opt<std::string> OutIncrement{
         "pass."),
     cl::init(""), cl::value_desc("R")};
 
-static cl::opt<bool> ExecuteInParallel{
+static cl::opt<unsigned int> ExecuteInParallel{
     "parallel-exec",
-    cl::desc("Enable launching input commands in parallel mode"),
-    cl::init(false)};
+    cl::Optional,
+    cl::init(1),
+    cl::desc("Specify the number of threads for launching input commands in "
+             "parallel mode"),
+};
 
 static void error(const Twine &Msg) {
   errs() << "llvm-foreach: " << Msg << '\n';
@@ -85,6 +88,31 @@ static void error(std::error_code EC, const Twine &Prefix) {
     error(Prefix + ": " + EC.message());
 }
 
+// With WaitUntilTerminates=false this function just goes through the all
+// submitted jobs to check if one of them has finished.
+// TODO: give a proper naming as this function doesn't wait for jobs with
+// WaitUntilTerminates=false
+void waitJobsToBeFinished(std::list<sys::ProcessInfo> &JobsSubmitted, int &Res,
+                          bool WaitUntilTerminates = true) {
+  std::string ErrMsg;
+  auto It = JobsSubmitted.begin();
+  while (It != JobsSubmitted.end()) {
+    sys::ProcessInfo WaitResult =
+        sys::Wait(*It, 0, /*WaitUntilTerminates*/ WaitUntilTerminates, &ErrMsg);
+
+    // Check if the job has finished (PID will be 0 if it's not).
+    if (!WaitUntilTerminates && !WaitResult.Pid) {
+      It++;
+      continue;
+    }
+    if (WaitResult.ReturnCode != 0) {
+      errs() << "llvm-foreach: " << ErrMsg << '\n';
+      Res = WaitResult.ReturnCode;
+    }
+    It = JobsSubmitted.erase(It);
+  }
+}
+
 int main(int argc, char **argv) {
   cl::ParseCommandLineOptions(
       argc, argv,
@@ -166,6 +194,13 @@ int main(int argc, char **argv) {
     PrevNumOfLines = FileLists[i].size();
   }
 
+  if (!ExecuteInParallel)
+    error("Number of parallel threads should be a positive integer");
+  if (ExecuteInParallel > 16) {
+    ExecuteInParallel = 16;
+    errs() << "llvm-foreach: adjusted number of threads to 16 (max available)";
+  }
+
   std::error_code EC;
   raw_fd_ostream OS{OutputFileList, EC, sys::fs::OpenFlags::OF_None};
   if (!OutputFileList.empty())
@@ -176,7 +211,7 @@ int main(int argc, char **argv) {
   std::string IncOutArg;
   std::vector<std::string> ResInArgs(InReplaceArgs.size());
   std::string ResFileList = "";
-  std::list<sys::ProcessInfo> CommandsStarted;
+  std::list<sys::ProcessInfo> JobsSubmitted;
   for (size_t j = 0; j != FileLists[0].size(); ++j) {
     for (size_t i = 0; i < InReplaceArgs.size(); ++i) {
       ArgumentReplace CurReplace = InReplaceArgs[i];
@@ -228,34 +263,17 @@ int main(int argc, char **argv) {
       Args[OutIncrementArg.ArgNum] = IncOutArg;
     }
 
-    std::string ErrMsg;
-    if (ExecuteInParallel) {
-      CommandsStarted.emplace_back(sys::ExecuteNoWait(
-          Prog, Args, /*Env=*/None, /*Redirects=*/None, /*MemoryLimit=*/0));
-      continue;
-    }
+    // Do not start execution of a new job until previous one(s) are finished,
+    // if the maximum number of parallel workers is reached.
+    while (JobsSubmitted.size() == ExecuteInParallel)
+      waitJobsToBeFinished(JobsSubmitted, Res, /*WaitUntilTerminates*/ false);
 
-    int Result =
-        sys::ExecuteAndWait(Prog, Args, /*Env=*/None, /*Redirects=*/None,
-                            /*SecondsToWait=*/0, /*MemoryLimit=*/0, &ErrMsg);
-    if (Result != 0) {
-      errs() << "llvm-foreach: " << ErrMsg << '\n';
-      Res = Result;
-    }
+    JobsSubmitted.emplace_back(sys::ExecuteNoWait(
+        Prog, Args, /*Env=*/None, /*Redirects=*/None, /*MemoryLimit=*/0));
   }
 
   // Wait for all commands to be executed.
-  std::string ErrMsg;
-  auto It = CommandsStarted.begin();
-  while (It != CommandsStarted.end()) {
-    sys::ProcessInfo WaitResult =
-        sys::Wait(*It, 0, /*WaitUntilTerminates*/ true, &ErrMsg);
-    if (WaitResult.ReturnCode != 0) {
-      errs() << "llvm-foreach: " << ErrMsg << '\n';
-      Res = WaitResult.ReturnCode;
-    }
-    It = CommandsStarted.erase(It);
-  }
+  waitJobsToBeFinished(JobsSubmitted, Res, /*WaitUntilTerminates*/ true);
 
   if (!OutputFileList.empty()) {
     OS.close();

From 82b17796e23c3370d21d35f08a1f95ed09438b98 Mon Sep 17 00:00:00 2001
From: Viktoria Maksimova <viktoria.maksimova@intel.com>
Date: Thu, 19 Aug 2021 18:31:14 +0300
Subject: [PATCH 03/11] Enable max 4 work threads by default (testing purpose)

---
 llvm/tools/llvm-foreach/llvm-foreach.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/tools/llvm-foreach/llvm-foreach.cpp b/llvm/tools/llvm-foreach/llvm-foreach.cpp
index 1728c8511a496..3415142fc4dc8 100644
--- a/llvm/tools/llvm-foreach/llvm-foreach.cpp
+++ b/llvm/tools/llvm-foreach/llvm-foreach.cpp
@@ -73,7 +73,7 @@ static cl::opt<std::string> OutIncrement{
 static cl::opt<unsigned int> ExecuteInParallel{
     "parallel-exec",
     cl::Optional,
-    cl::init(1),
+    cl::init(4),
     cl::desc("Specify the number of threads for launching input commands in "
              "parallel mode"),
 };

From 516779bfae05eb333e4d820ebb1a10fa0c35093a Mon Sep 17 00:00:00 2001
From: Viktoria Maksimova <viktoria.maksimova@intel.com>
Date: Thu, 19 Aug 2021 19:05:52 +0300
Subject: [PATCH 04/11] Fix llvm-foreach/retain-return-val.c test

---
 llvm/test/tools/llvm-foreach/retain-return-val.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/test/tools/llvm-foreach/retain-return-val.c b/llvm/test/tools/llvm-foreach/retain-return-val.c
index f70d53313b6c1..0dd6570788b40 100644
--- a/llvm/test/tools/llvm-foreach/retain-return-val.c
+++ b/llvm/test/tools/llvm-foreach/retain-return-val.c
@@ -18,6 +18,6 @@
 // RUN: chmod 777 %t2.sh
 // RUN: %t2.sh
 // RUN: FileCheck < %t.res %s
-// CHECK: Content of first file
-// CHECK: Content of second file
+// CHECK-DAG: Content of first file
+// CHECK-DAG: Content of second file
 // CHECK: 21

From f4051c0d29bd1ac9dbdc146fe1358e39f35ab056 Mon Sep 17 00:00:00 2001
From: Viktoria Maksimova <viktoria.maksimova@intel.com>
Date: Mon, 30 Aug 2021 12:23:29 +0300
Subject: [PATCH 05/11] Limit the maximum number of threads to max safe
 available on the hardware

---
 llvm/tools/llvm-foreach/llvm-foreach.cpp | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/llvm/tools/llvm-foreach/llvm-foreach.cpp b/llvm/tools/llvm-foreach/llvm-foreach.cpp
index 3415142fc4dc8..81d98f1e0144f 100644
--- a/llvm/tools/llvm-foreach/llvm-foreach.cpp
+++ b/llvm/tools/llvm-foreach/llvm-foreach.cpp
@@ -18,6 +18,7 @@
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
 #include "llvm/Support/SystemUtils.h"
+#include "llvm/Support/Threading.h"
 
 #include <list>
 #include <vector>
@@ -196,9 +197,12 @@ int main(int argc, char **argv) {
 
   if (!ExecuteInParallel)
     error("Number of parallel threads should be a positive integer");
-  if (ExecuteInParallel > 16) {
-    ExecuteInParallel = 16;
-    errs() << "llvm-foreach: adjusted number of threads to 16 (max available)";
+
+  size_t MaxSafeNumThreads = optimal_concurrency().compute_thread_count();
+  if (ExecuteInParallel > MaxSafeNumThreads) {
+    ExecuteInParallel = MaxSafeNumThreads;
+    outs() << "llvm-foreach: adjusted number of threads to "
+           << MaxSafeNumThreads << " (max safe available).\n";
   }
 
   std::error_code EC;

From 8e709ba54c6b475ac97bbeba94e39bafa7c300b7 Mon Sep 17 00:00:00 2001
From: Viktoria Maksimova <viktoria.maksimova@intel.com>
Date: Tue, 31 Aug 2021 16:55:08 +0300
Subject: [PATCH 06/11] Apply improvements after code review

---
 .../tools/llvm-foreach/llvm-foreach-lin.ll    | 27 +++++++------
 .../tools/llvm-foreach/llvm-foreach-win.ll    | 27 +++++++------
 llvm/tools/llvm-foreach/llvm-foreach.cpp      | 39 ++++++++++---------
 3 files changed, 51 insertions(+), 42 deletions(-)

diff --git a/llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll b/llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll
index 1608bd945d9b0..d6d70a8547b8b 100644
--- a/llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll
+++ b/llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll
@@ -4,16 +4,19 @@
 ; RUN: echo 'Content of second file' > %t2.tgt
 ; RUN: echo "%t1.tgt" > %t.list
 ; RUN: echo "%t2.tgt" >> %t.list
-; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
-; RUN: FileCheck < %t.res %s
-; RUN: llvm-foreach --parallel-exec=2 --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
+; RUN: llvm-foreach --jobs=2 --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
 ; RUN: FileCheck < %t.res %s
 ; CHECK-DAG: [[FIRST:.+1.tgt]]
 ; CHECK-DAG: [[SECOND:.+2.tgt]]
-;
+
+; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
+; RUN: FileCheck < %t.res %s --check-prefix=CHECK-ORDER
+; CHECK-ORDER: [[FIRST:.+1.tgt]]
+; CHECK-ORDER: [[SECOND:.+2.tgt]]
+
 ; RUN: llvm-foreach --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- cp "{}" %t
 ; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST
-; RUN: llvm-foreach --parallel-exec=2 --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- cp "{}" %t
+; RUN: llvm-foreach --jobs=2 --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- cp "{}" %t
 ; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST
 ; CHECK-LIST-DAG: [[FIRST:.+\.out]]
 ; CHECK-LIST-DAG: [[SECOND:.+\.out]]
@@ -26,7 +29,7 @@
 ; RUN: echo "%t4.tgt" >> %t1.list
 ; RUN: llvm-foreach --in-replace="{}" --in-replace="inrep" --in-file-list=%t.list --in-file-list=%t1.list --out-increment="%t_out.prj" -- echo -first-part-of-arg={}.out -first-part-of-arg=inrep.out -another-arg=%t_out.prj > %t1.res
 ; RUN: FileCheck < %t1.res %s --check-prefix=CHECK-DOUBLE-LISTS
-; RUN: llvm-foreach --parallel-exec=2 --in-replace="{}" --in-replace="inrep" --in-file-list=%t.list --in-file-list=%t1.list --out-increment="%t_out.prj" -- echo -first-part-of-arg={}.out -first-part-of-arg=inrep.out -another-arg=%t_out.prj > %t1.res
+; RUN: llvm-foreach --jobs=2 --in-replace="{}" --in-replace="inrep" --in-file-list=%t.list --in-file-list=%t1.list --out-increment="%t_out.prj" -- echo -first-part-of-arg={}.out -first-part-of-arg=inrep.out -another-arg=%t_out.prj > %t1.res
 ; RUN: FileCheck < %t1.res %s --check-prefix=CHECK-DOUBLE-LISTS
 ; CHECK-DOUBLE-LISTS-DAG: -first-part-of-arg=[[FIRST:.+1.tgt.out]] -first-part-of-arg=[[THIRD:.+3.tgt.out]] -another-arg={{.+}}_out.prj
 ; CHECK-DOUBLE-LISTS-DAG: -first-part-of-arg=[[SECOND:.+2.tgt.out]] -first-part-of-arg=[[FOURTH:.+4.tgt.out]] -another-arg={{.+}}_out.prj_1
@@ -35,9 +38,9 @@
 ; RUN: echo "%t2.tgt" >> %t2.list
 ; RUN: echo "%t3.tgt" >> %t2.list
 ; RUN: echo "%t4.tgt" >> %t2.list
-; RUN: llvm-foreach --parallel-exec=2 --in-replace="{}" --in-file-list=%t2.list -- echo "{}" > %t2.res
-; RUN: FileCheck < %t2.res %s --check-prefix=CHECK-PARALLEL-EXEC
-; CHECK-PARALLEL-EXEC-DAG: [[FIRST:.+1.tgt]]
-; CHECK-PARALLEL-EXEC-DAG: [[SECOND:.+2.tgt]]
-; CHECK-PARALLEL-EXEC-DAG: [[THIRD:.+3.tgt]]
-; CHECK-PARALLEL-EXEC-DAG: [[FOURTH:.+4.tgt]]
+; RUN: llvm-foreach -j 2 --in-replace="{}" --in-file-list=%t2.list -- echo "{}" > %t2.res
+; RUN: FileCheck < %t2.res %s --check-prefix=CHECK-PARALLEL-JOBS
+; CHECK-PARALLEL-JOBS-DAG: [[FIRST:.+1.tgt]]
+; CHECK-PARALLEL-JOBS-DAG: [[SECOND:.+2.tgt]]
+; CHECK-PARALLEL-JOBS-DAG: [[THIRD:.+3.tgt]]
+; CHECK-PARALLEL-JOBS-DAG: [[FOURTH:.+4.tgt]]
diff --git a/llvm/test/tools/llvm-foreach/llvm-foreach-win.ll b/llvm/test/tools/llvm-foreach/llvm-foreach-win.ll
index dd6bef14b4d6a..0b6953393d90c 100644
--- a/llvm/test/tools/llvm-foreach/llvm-foreach-win.ll
+++ b/llvm/test/tools/llvm-foreach/llvm-foreach-win.ll
@@ -4,16 +4,19 @@
 ; RUN: echo 'Content of second file' > %t2.tgt
 ; RUN: echo "%t1.tgt" > %t.list
 ; RUN: echo "%t2.tgt" >> %t.list
-; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
-; RUN: FileCheck < %t.res %s
-; RUN: llvm-foreach --parallel-exec=2 --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
+; RUN: llvm-foreach --jobs=2 --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
 ; RUN: FileCheck < %t.res %s
 ; CHECK-DAG: [[FIRST:.+1.tgt]]
 ; CHECK-DAG: [[SECOND:.+2.tgt]]
-;
+
+; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.list -- echo "{}" > %t.res
+; RUN: FileCheck < %t.res %s --check-prefix=CHECK-ORDER
+; CHECK-ORDER: [[FIRST:.+1.tgt]]
+; CHECK-ORDER: [[SECOND:.+2.tgt]]
+
 ; RUN: llvm-foreach --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- xcopy /y "{}" %t
 ; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST
-; RUN: llvm-foreach --parallel-exec=2 --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- xcopy /y "{}" %t
+; RUN: llvm-foreach --jobs=2 --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- xcopy /y "{}" %t
 ; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST
 ; CHECK-LIST-DAG: [[FIRST:.+\.out]]
 ; CHECK-LIST-DAG: [[SECOND:.+\.out]]
@@ -26,7 +29,7 @@
 ; RUN: echo "%t4.tgt" >> %t1.list
 ; RUN: llvm-foreach --in-replace="{}" --in-replace="inrep" --in-file-list=%t.list --in-file-list=%t1.list --out-increment=%t_out.prj -- echo -first-part-of-arg={}.out -first-part-of-arg=inrep.out -another-arg=%t_out.prj > %t1.res
 ; RUN: FileCheck < %t1.res %s --check-prefix=CHECK-DOUBLE-LISTS
-; RUN: llvm-foreach --parallel-exec=2 --in-replace="{}" --in-replace="inrep" --in-file-list=%t.list --in-file-list=%t1.list --out-increment=%t_out.prj -- echo -first-part-of-arg={}.out -first-part-of-arg=inrep.out -another-arg=%t_out.prj > %t1.res
+; RUN: llvm-foreach --jobs=2 --in-replace="{}" --in-replace="inrep" --in-file-list=%t.list --in-file-list=%t1.list --out-increment=%t_out.prj -- echo -first-part-of-arg={}.out -first-part-of-arg=inrep.out -another-arg=%t_out.prj > %t1.res
 ; RUN: FileCheck < %t1.res %s --check-prefix=CHECK-DOUBLE-LISTS
 ; CHECK-DOUBLE-LISTS-DAG: -first-part-of-arg=[[FIRST:.+1.tgt.out]] -first-part-of-arg=[[THIRD:.+3.tgt.out]] -another-arg={{.+}}_out.prj
 ; CHECK-DOUBLE-LISTS-DAG: -first-part-of-arg=[[SECOND:.+2.tgt.out]] -first-part-of-arg=[[FOURTH:.+4.tgt.out]] -another-arg={{.+}}_out.prj_1
@@ -35,9 +38,9 @@
 ; RUN: echo "%t2.tgt" >> %t2.list
 ; RUN: echo "%t3.tgt" >> %t2.list
 ; RUN: echo "%t4.tgt" >> %t2.list
-; RUN: llvm-foreach --parallel-exec=2 --in-replace="{}" --in-file-list=%t2.list -- echo "{}" > %t2.res
-; RUN: FileCheck < %t2.res %s --check-prefix=CHECK-PARALLEL-EXEC
-; CHECK-PARALLEL-EXEC-DAG: [[FIRST:.+1.tgt]]
-; CHECK-PARALLEL-EXEC-DAG: [[SECOND:.+2.tgt]]
-; CHECK-PARALLEL-EXEC-DAG: [[THIRD:.+3.tgt]]
-; CHECK-PARALLEL-EXEC-DAG: [[FOURTH:.+4.tgt]]
+; RUN: llvm-foreach --jobs=2 --in-replace="{}" --in-file-list=%t2.list -- echo "{}" > %t2.res
+; RUN: FileCheck < %t2.res %s --check-prefix=CHECK-PARALLEL-JOBS
+; CHECK-PARALLEL-JOBS-DAG: [[FIRST:.+1.tgt]]
+; CHECK-PARALLEL-JOBS-DAG: [[SECOND:.+2.tgt]]
+; CHECK-PARALLEL-JOBS-DAG: [[THIRD:.+3.tgt]]
+; CHECK-PARALLEL-JOBS-DAG: [[FOURTH:.+4.tgt]]
diff --git a/llvm/tools/llvm-foreach/llvm-foreach.cpp b/llvm/tools/llvm-foreach/llvm-foreach.cpp
index 81d98f1e0144f..0379061596cd5 100644
--- a/llvm/tools/llvm-foreach/llvm-foreach.cpp
+++ b/llvm/tools/llvm-foreach/llvm-foreach.cpp
@@ -71,14 +71,17 @@ static cl::opt<std::string> OutIncrement{
         "pass."),
     cl::init(""), cl::value_desc("R")};
 
-static cl::opt<unsigned int> ExecuteInParallel{
-    "parallel-exec",
+static cl::opt<unsigned int> JobsInParallel{
+    "jobs",
     cl::Optional,
-    cl::init(4),
+    cl::init(1),
     cl::desc("Specify the number of threads for launching input commands in "
              "parallel mode"),
 };
 
+static cl::alias JobsInParallelShort{"j", cl::desc("Alias for --jobs"),
+                                     cl::aliasopt(JobsInParallel)};
+
 static void error(const Twine &Msg) {
   errs() << "llvm-foreach: " << Msg << '\n';
   exit(1);
@@ -89,29 +92,29 @@ static void error(std::error_code EC, const Twine &Prefix) {
     error(Prefix + ": " + EC.message());
 }
 
-// With WaitUntilTerminates=false this function just goes through the all
+// With BlockingWait=false this function just goes through the all
 // submitted jobs to check if one of them has finished.
-// TODO: give a proper naming as this function doesn't wait for jobs with
-// WaitUntilTerminates=false
-void waitJobsToBeFinished(std::list<sys::ProcessInfo> &JobsSubmitted, int &Res,
-                          bool WaitUntilTerminates = true) {
+int checkIfJobsAreFinished(std::list<sys::ProcessInfo> &JobsSubmitted,
+                           bool BlockingWait = true) {
   std::string ErrMsg;
   auto It = JobsSubmitted.begin();
   while (It != JobsSubmitted.end()) {
     sys::ProcessInfo WaitResult =
-        sys::Wait(*It, 0, /*WaitUntilTerminates*/ WaitUntilTerminates, &ErrMsg);
+        sys::Wait(*It, 0, /*WaitUntilTerminates*/ BlockingWait, &ErrMsg);
 
     // Check if the job has finished (PID will be 0 if it's not).
-    if (!WaitUntilTerminates && !WaitResult.Pid) {
+    if (!BlockingWait && !WaitResult.Pid) {
       It++;
       continue;
     }
+    It = JobsSubmitted.erase(It);
+
     if (WaitResult.ReturnCode != 0) {
       errs() << "llvm-foreach: " << ErrMsg << '\n';
-      Res = WaitResult.ReturnCode;
+      return WaitResult.ReturnCode;
     }
-    It = JobsSubmitted.erase(It);
   }
+  return 0;
 }
 
 int main(int argc, char **argv) {
@@ -195,12 +198,12 @@ int main(int argc, char **argv) {
     PrevNumOfLines = FileLists[i].size();
   }
 
-  if (!ExecuteInParallel)
+  if (!JobsInParallel)
     error("Number of parallel threads should be a positive integer");
 
   size_t MaxSafeNumThreads = optimal_concurrency().compute_thread_count();
-  if (ExecuteInParallel > MaxSafeNumThreads) {
-    ExecuteInParallel = MaxSafeNumThreads;
+  if (JobsInParallel > MaxSafeNumThreads) {
+    JobsInParallel = MaxSafeNumThreads;
     outs() << "llvm-foreach: adjusted number of threads to "
            << MaxSafeNumThreads << " (max safe available).\n";
   }
@@ -269,15 +272,15 @@ int main(int argc, char **argv) {
 
     // Do not start execution of a new job until previous one(s) are finished,
     // if the maximum number of parallel workers is reached.
-    while (JobsSubmitted.size() == ExecuteInParallel)
-      waitJobsToBeFinished(JobsSubmitted, Res, /*WaitUntilTerminates*/ false);
+    while (JobsSubmitted.size() == JobsInParallel)
+      Res = checkIfJobsAreFinished(JobsSubmitted, /*BlockingWait*/ false);
 
     JobsSubmitted.emplace_back(sys::ExecuteNoWait(
         Prog, Args, /*Env=*/None, /*Redirects=*/None, /*MemoryLimit=*/0));
   }
 
   // Wait for all commands to be executed.
-  waitJobsToBeFinished(JobsSubmitted, Res, /*WaitUntilTerminates*/ true);
+  Res = checkIfJobsAreFinished(JobsSubmitted, /*BlockingWait*/ true);
 
   if (!OutputFileList.empty()) {
     OS.close();

From 8a9f664840c5ec0538118c7702c1a6412b5dbc4a Mon Sep 17 00:00:00 2001
From: Viktoria Maximova <viktoria.maksimova@intel.com>
Date: Tue, 31 Aug 2021 17:52:17 +0300
Subject: [PATCH 07/11] Update llvm/tools/llvm-foreach/llvm-foreach.cpp

Co-authored-by: Alexey Sachkov <alexey.sachkov@intel.com>
---
 llvm/tools/llvm-foreach/llvm-foreach.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/tools/llvm-foreach/llvm-foreach.cpp b/llvm/tools/llvm-foreach/llvm-foreach.cpp
index 0379061596cd5..ece1084f221b3 100644
--- a/llvm/tools/llvm-foreach/llvm-foreach.cpp
+++ b/llvm/tools/llvm-foreach/llvm-foreach.cpp
@@ -93,7 +93,7 @@ static void error(std::error_code EC, const Twine &Prefix) {
 }
 
 // With BlockingWait=false this function just goes through the all
-// submitted jobs to check if one of them has finished.
+// submitted jobs to check if some of them have finished.
 int checkIfJobsAreFinished(std::list<sys::ProcessInfo> &JobsSubmitted,
                            bool BlockingWait = true) {
   std::string ErrMsg;

From 1690ad7ac36327783e3ff84c1a4ea48c61e06ce4 Mon Sep 17 00:00:00 2001
From: Viktoria Maksimova <viktoria.maksimova@intel.com>
Date: Fri, 3 Sep 2021 13:36:47 +0300
Subject: [PATCH 08/11] Apply suggestions

---
 llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll | 4 ++--
 llvm/tools/llvm-foreach/llvm-foreach.cpp         | 9 +++++++--
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll b/llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll
index d6d70a8547b8b..beac4a88c6e05 100644
--- a/llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll
+++ b/llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll
@@ -18,8 +18,8 @@
 ; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST
 ; RUN: llvm-foreach --jobs=2 --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- cp "{}" %t
 ; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST
-; CHECK-LIST-DAG: [[FIRST:.+\.out]]
-; CHECK-LIST-DAG: [[SECOND:.+\.out]]
+; CHECK-LIST: [[FIRST:.+\.out]]
+; CHECK-LIST: [[SECOND:.+\.out]]
 ; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.out.list -- FileCheck --input-file="{}" %s --check-prefix=CHECK-CONTENT
 ; CHECK-CONTENT: Content of
 
diff --git a/llvm/tools/llvm-foreach/llvm-foreach.cpp b/llvm/tools/llvm-foreach/llvm-foreach.cpp
index ece1084f221b3..8f064db7b3fd5 100644
--- a/llvm/tools/llvm-foreach/llvm-foreach.cpp
+++ b/llvm/tools/llvm-foreach/llvm-foreach.cpp
@@ -107,6 +107,7 @@ int checkIfJobsAreFinished(std::list<sys::ProcessInfo> &JobsSubmitted,
       It++;
       continue;
     }
+    assert(BlockingWait || WaitResult.Pid);
     It = JobsSubmitted.erase(It);
 
     if (WaitResult.ReturnCode != 0) {
@@ -273,14 +274,18 @@ int main(int argc, char **argv) {
     // Do not start execution of a new job until previous one(s) are finished,
     // if the maximum number of parallel workers is reached.
     while (JobsSubmitted.size() == JobsInParallel)
-      Res = checkIfJobsAreFinished(JobsSubmitted, /*BlockingWait*/ false);
+      if (int Result =
+              checkIfJobsAreFinished(JobsSubmitted, /*BlockingWait*/ false))
+        Res = Result;
 
     JobsSubmitted.emplace_back(sys::ExecuteNoWait(
         Prog, Args, /*Env=*/None, /*Redirects=*/None, /*MemoryLimit=*/0));
   }
 
   // Wait for all commands to be executed.
-  Res = checkIfJobsAreFinished(JobsSubmitted, /*BlockingWait*/ true);
+  if (int Result = checkIfJobsAreFinished(JobsSubmitted, /*BlockingWait*/ true))
+    Res = Result;
+  assert(JobsSubmitted.empty());
 
   if (!OutputFileList.empty()) {
     OS.close();

From 2edab5f3db095bba5b9181e2514cf534ef5b64de Mon Sep 17 00:00:00 2001
From: Viktoria Maksimova <viktoria.maksimova@intel.com>
Date: Fri, 3 Sep 2021 23:20:26 +0300
Subject: [PATCH 09/11] Update test cases and fix commands waiting

---
 llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll |  6 ++++--
 llvm/test/tools/llvm-foreach/llvm-foreach-win.ll | 10 ++++++----
 llvm/tools/llvm-foreach/llvm-foreach.cpp         |  7 ++++---
 3 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll b/llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll
index beac4a88c6e05..7aeaa800db13b 100644
--- a/llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll
+++ b/llvm/test/tools/llvm-foreach/llvm-foreach-lin.ll
@@ -20,8 +20,10 @@
 ; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST
 ; CHECK-LIST: [[FIRST:.+\.out]]
 ; CHECK-LIST: [[SECOND:.+\.out]]
-; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.out.list -- FileCheck --input-file="{}" %s --check-prefix=CHECK-CONTENT
-; CHECK-CONTENT: Content of
+; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.out.list -- cat "{}" > %t.order
+; RUN: FileCheck < %t.order %s --check-prefix=CHECK-CONTENT
+; CHECK-CONTENT: Content of first file
+; CHECK-CONTENT-NEXT: Content of second file
 
 ; RUN: echo 'something' > %t3.tgt
 ; RUN: echo 'something again' > %t4.tgt
diff --git a/llvm/test/tools/llvm-foreach/llvm-foreach-win.ll b/llvm/test/tools/llvm-foreach/llvm-foreach-win.ll
index 0b6953393d90c..d6a12c45b1885 100644
--- a/llvm/test/tools/llvm-foreach/llvm-foreach-win.ll
+++ b/llvm/test/tools/llvm-foreach/llvm-foreach-win.ll
@@ -18,10 +18,12 @@
 ; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST
 ; RUN: llvm-foreach --jobs=2 --in-replace="{}" --out-replace=%t --out-ext=out --in-file-list=%t.list --out-file-list=%t.out.list -- xcopy /y "{}" %t
 ; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST
-; CHECK-LIST-DAG: [[FIRST:.+\.out]]
-; CHECK-LIST-DAG: [[SECOND:.+\.out]]
-; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.out.list -- FileCheck --input-file="{}" %s --check-prefix=CHECK-CONTENT
-; CHECK-CONTENT: Content of
+; CHECK-LIST: [[FIRST:.+\.out]]
+; CHECK-LIST: [[SECOND:.+\.out]]
+; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.out.list -- type "{}" > %t.order
+; RUN: FileCheck < %t.order %s --check-prefix=CHECK-CONTENT
+; CHECK-CONTENT: Content of first file
+; CHECK-CONTENT-NEXT: Content of second file
 
 ; RUN: echo 'something' > %t3.tgt
 ; RUN: echo 'something again' > %t4.tgt
diff --git a/llvm/tools/llvm-foreach/llvm-foreach.cpp b/llvm/tools/llvm-foreach/llvm-foreach.cpp
index 8f064db7b3fd5..11d6b24af693e 100644
--- a/llvm/tools/llvm-foreach/llvm-foreach.cpp
+++ b/llvm/tools/llvm-foreach/llvm-foreach.cpp
@@ -283,9 +283,10 @@ int main(int argc, char **argv) {
   }
 
   // Wait for all commands to be executed.
-  if (int Result = checkIfJobsAreFinished(JobsSubmitted, /*BlockingWait*/ true))
-    Res = Result;
-  assert(JobsSubmitted.empty());
+  while (!JobsSubmitted.empty())
+    if (int Result =
+            checkIfJobsAreFinished(JobsSubmitted, /*BlockingWait*/ true))
+      Res = Result;
 
   if (!OutputFileList.empty()) {
     OS.close();

From 6f9748e10068d219658fc809967ac2003e631205 Mon Sep 17 00:00:00 2001
From: Viktoria Maksimova <viktoria.maksimova@intel.com>
Date: Mon, 6 Sep 2021 12:52:17 +0300
Subject: [PATCH 10/11] Remove unneeded -DAG checks

---
 llvm/test/tools/llvm-foreach/retain-return-val.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/test/tools/llvm-foreach/retain-return-val.c b/llvm/test/tools/llvm-foreach/retain-return-val.c
index 0dd6570788b40..f70d53313b6c1 100644
--- a/llvm/test/tools/llvm-foreach/retain-return-val.c
+++ b/llvm/test/tools/llvm-foreach/retain-return-val.c
@@ -18,6 +18,6 @@
 // RUN: chmod 777 %t2.sh
 // RUN: %t2.sh
 // RUN: FileCheck < %t.res %s
-// CHECK-DAG: Content of first file
-// CHECK-DAG: Content of second file
+// CHECK: Content of first file
+// CHECK: Content of second file
 // CHECK: 21

From 6cd8cb2b81e545dbee7b6bef7efd0b7e22e6ef57 Mon Sep 17 00:00:00 2001
From: Viktoria Maksimova <viktoria.maksimova@intel.com>
Date: Tue, 7 Sep 2021 18:10:08 +0300
Subject: [PATCH 11/11] Fix test failure on Windows

---
 llvm/test/tools/llvm-foreach/llvm-foreach-win.ll | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/test/tools/llvm-foreach/llvm-foreach-win.ll b/llvm/test/tools/llvm-foreach/llvm-foreach-win.ll
index d6a12c45b1885..0072ab71956f4 100644
--- a/llvm/test/tools/llvm-foreach/llvm-foreach-win.ll
+++ b/llvm/test/tools/llvm-foreach/llvm-foreach-win.ll
@@ -20,7 +20,7 @@
 ; RUN: FileCheck < %t.out.list %s --check-prefix=CHECK-LIST
 ; CHECK-LIST: [[FIRST:.+\.out]]
 ; CHECK-LIST: [[SECOND:.+\.out]]
-; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.out.list -- type "{}" > %t.order
+; RUN: llvm-foreach --in-replace="{}" --in-file-list=%t.out.list -- cat "{}" > %t.order
 ; RUN: FileCheck < %t.order %s --check-prefix=CHECK-CONTENT
 ; CHECK-CONTENT: Content of first file
 ; CHECK-CONTENT-NEXT: Content of second file