[BuildBot] Improve usability of buildbot scripts (#1472)

1. Make all parameters optional with (hopefully) useful defaults.
2. Run actions through CMake to make the independent from the build
system (e.g. Ninja/Makefiles/MSVC/etc.)

Signed-off-by: Alexey Bader <[email protected]>
Co-Authored-By: Dmitry Vodopyanov <[email protected]>

Author: bader

buildbot/check.py

@@ -1,24 +1,41 @@
 import argparse
 import os
+import multiprocessing
 import subprocess
 import sys
 
 DEFAULT_CPU_COUNT = 4
 
 def do_check(args):
-    ret = False
-
-    cpu_count = os.cpu_count()
-    if cpu_count is None:
+    try:
+        cpu_count = multiprocessing.cpu_count()
+    except NotImplementedError:
         cpu_count = DEFAULT_CPU_COUNT
 
+    # Get absolute path to source directory
+    if args.src_dir:
+      abs_src_dir = os.path.abspath(args.src_dir)
+    else:
+      abs_src_dir = os.path.abspath(os.path.join(__file__, "../.."))
+    # Get absolute path to build directory
+    if args.obj_dir:
+      abs_obj_dir = os.path.abspath(args.obj_dir)
+    else:
+      abs_obj_dir = os.path.join(abs_src_dir, "build")
+
+    cmake_cmd = [
+        "cmake",
+        "--build", abs_obj_dir,
+        "--",
+        args.test_suite,
+        "-j", str(cpu_count)]
+
+    print(cmake_cmd)
+
     env_tmp=os.environ
     env_tmp["LIT_ARGS"]="\"{}\"".format("-v")
 
-    make_cmd = ["ninja", args.test_suite, "-j", str(cpu_count)]
-    print(make_cmd)
-
-    subprocess.check_call(make_cmd, cwd=args.obj_dir, env=env_tmp)
+    subprocess.check_call(cmake_cmd, cwd=abs_obj_dir, env=env_tmp)
 
     ret = True
     return ret
@@ -34,8 +51,8 @@ def main():
     parser.add_argument("-w", "--builder-dir", metavar="BUILDER_DIR",
                         help="builder directory, which is the directory contains source and build directories")
     parser.add_argument("-s", "--src-dir", metavar="SRC_DIR", help="source directory")
-    parser.add_argument("-o", "--obj-dir", metavar="OBJ_DIR", required=True, help="build directory")
-    parser.add_argument("-t", "--test-suite", metavar="TEST_SUITE", required=True, help="check-xxx target")
+    parser.add_argument("-o", "--obj-dir", metavar="OBJ_DIR", help="build directory")
+    parser.add_argument("-t", "--test-suite", metavar="TEST_SUITE", default="check-all", help="check-xxx target")
 
     args = parser.parse_args()
 
@@ -47,4 +64,3 @@ def main():
     ret = main()
     exit_code = 0 if ret else 1
     sys.exit(exit_code)
-

buildbot/compile.py

@@ -2,6 +2,7 @@
 import multiprocessing
 import subprocess
 import sys
+import os
 
 DEFAULT_CPU_COUNT = 4
 
@@ -12,10 +13,28 @@ def do_compile(args):
     except NotImplementedError:
         cpu_count = DEFAULT_CPU_COUNT
 
-    make_cmd = ["ninja", "-j", str(cpu_count), "deploy-sycl-toolchain", "deploy-opencl-aot"]
-    print(make_cmd)
+    # Get absolute path to source directory
+    if args.src_dir:
+      abs_src_dir = os.path.abspath(args.src_dir)
+    else:
+      abs_src_dir = os.path.abspath(os.path.join(__file__, "../.."))
+    # Get absolute path to build directory
+    if args.obj_dir:
+      abs_obj_dir = os.path.abspath(args.obj_dir)
+    else:
+      abs_obj_dir = os.path.join(abs_src_dir, "build")
 
-    subprocess.check_call(make_cmd, cwd=args.obj_dir)
+    cmake_cmd = [
+        "cmake",
+        "--build", abs_obj_dir,
+        "--",
+        "deploy-sycl-toolchain",
+        "deploy-opencl-aot",
+        "-j", str(cpu_count)]
+
+    print(cmake_cmd)
+
+    subprocess.check_call(cmake_cmd, cwd=abs_obj_dir)
 
     return True
 
@@ -31,7 +50,7 @@ def main():
     parser.add_argument("-w", "--builder-dir", metavar="BUILDER_DIR",
                         help="builder directory, which is the directory contains source and build directories")
     parser.add_argument("-s", "--src-dir", metavar="SRC_DIR", help="source directory")
-    parser.add_argument("-o", "--obj-dir", metavar="OBJ_DIR", required=True, help="build directory")
+    parser.add_argument("-o", "--obj-dir", metavar="OBJ_DIR", help="build directory")
 
     args = parser.parse_args()
 
@@ -44,4 +63,3 @@ def main():
     ret = main()
     exit_code = 0 if ret else 1
     sys.exit(exit_code)
-

buildbot/configure.py

@@ -4,16 +4,21 @@
 import sys
 import platform
 
-# TODO:
-# 1. Make all required options optional
-# 2. Create obj_dir from the script if it doesn't exist
-
 def do_configure(args):
     ret = False
 
-    # Get absolute paths
-    abs_src_dir = os.path.abspath(args.src_dir)
-    abs_obj_dir = os.path.abspath(args.obj_dir)
+    # Get absolute path to source directory
+    if args.src_dir:
+      abs_src_dir = os.path.abspath(args.src_dir)
+    else:
+      abs_src_dir = os.path.abspath(os.path.join(__file__, "../.."))
+    # Get absolute path to build directory
+    if args.obj_dir:
+      abs_obj_dir = os.path.abspath(args.obj_dir)
+    else:
+      abs_obj_dir = os.path.join(abs_src_dir, "build")
+    if not os.path.isdir(abs_obj_dir):
+      os.makedirs(abs_obj_dir)
 
     llvm_dir = os.path.join(abs_src_dir, "llvm")
     sycl_dir = os.path.join(abs_src_dir, "sycl")
@@ -60,7 +65,7 @@ def do_configure(args):
 
     cmake_cmd = [
         "cmake",
-        "-G", "Ninja",
+        "-G", args.cmake_gen,
         "-DCMAKE_BUILD_TYPE={}".format(args.build_type),
         "-DLLVM_ENABLE_ASSERTIONS={}".format(llvm_enable_assertions),
         "-DLLVM_TARGETS_TO_BUILD={}".format(llvm_targets_to_build),
@@ -117,17 +122,18 @@ def main():
     parser.add_argument("-r", "--pr-number", metavar="PR_NUM", help="pull request number")
     parser.add_argument("-w", "--builder-dir", metavar="BUILDER_DIR",
                         help="builder directory, which is the directory contains source and build directories")
-    parser.add_argument("-s", "--src-dir", metavar="SRC_DIR", required=True, help="source directory")
-    parser.add_argument("-o", "--obj-dir", metavar="OBJ_DIR", required=True, help="build directory")
+    parser.add_argument("-s", "--src-dir", metavar="SRC_DIR", help="source directory (autodetected by default)")
+    parser.add_argument("-o", "--obj-dir", metavar="OBJ_DIR", help="build directory. (<src>/build by default)")
     parser.add_argument("-t", "--build-type",
-                        metavar="BUILD_TYPE", required=True, help="build type, debug or release")
+                        metavar="BUILD_TYPE", default="Release", help="build type: Debug, Release")
     parser.add_argument("--cuda", action='store_true', help="switch from OpenCL to CUDA")
     parser.add_argument("--no-assertions", action='store_true', help="build without assertions")
     parser.add_argument("--docs", action='store_true', help="build Doxygen documentation")
     parser.add_argument("--system-ocl", action='store_true', help="use OpenCL deps from system (no download)")
     parser.add_argument("--no-werror", action='store_true', help="Don't treat warnings as errors")
     parser.add_argument("--shared-libs", action='store_true', help="Build shared libraries")
     parser.add_argument("--cmake-opt", action='append', help="Additional CMake option not configured via script parameters")
+    parser.add_argument("--cmake-gen", default="Ninja", help="CMake generator")
 
     args = parser.parse_args()
 

sycl/doc/GetStartedGuide.md

@@ -79,15 +79,15 @@ variables is in these files.
 **Linux**
 
 ```bash
-python $DPCPP_HOME/llvm/buildbot/configure.py -s $DPCPP_HOME/llvm -o $DPCPP_HOME/build -t release
-python $DPCPP_HOME/llvm/buildbot/compile.py -s $DPCPP_HOME/llvm -o $DPCPP_HOME/build
+python $DPCPP_HOME/llvm/buildbot/configure.py
+python $DPCPP_HOME/llvm/buildbot/compile.py
 ```
 
 **Windows**
 
 ```bat
-python %DPCPP_HOME%\llvm\buildbot\configure.py -s %DPCPP_HOME%\llvm -o %DPCPP_HOME%\build -t release
-python %DPCPP_HOME%\llvm\buildbot\compile.py -s %DPCPP_HOME%\llvm -o %DPCPP_HOME%\build
+python %DPCPP_HOME%\llvm\buildbot\configure.py
+python %DPCPP_HOME%\llvm\buildbot\compile.py
 ```
 
 **Options**
@@ -99,6 +99,8 @@ You can use the following flags with `configure.py`:
  * `--cuda` -> use the cuda backend (see [Nvidia CUDA](#build-dpc-toolchain-with-support-for-nvidia-cuda))
  * `--shared-libs` -> Build shared libraries
  * `-t` -> Build type (debug or release)
+ * `-o` -> Path to build directory
+ * `--cmake-gen` -> Set build system type (e.g. `--cmake-gen "Unix Makefiles"`)
 
 Ahead-of-time compilation for the Intel® processors is enabled by default.
 For more, see [opencl-aot documentation](../../opencl-aot/README.md).
@@ -260,9 +262,15 @@ c:\oclcpu_rt_<cpu_version>\install.bat c:\tbb_<tbb_version>\tbb\bin\intel64\vc14
 
 To verify that built DPC++ toolchain is working correctly, run:
 
-**Linux/Windows (64-bit)**
+**Linux**
 ```bash
-ninja check-all
+python $DPCPP_HOME/llvm/buildbot/check.py
+```
+
+**Windows**
+
+```bat
+python %DPCPP_HOME%\llvm\buildbot\check.py
 ```
 
 If no OpenCL GPU/CPU runtimes are available, the corresponding tests are