Merge pull request #41714 from lorentey/fix-update-checkout-output

[utils] Make update-checkout output more readable

Author: lorentey

utils/check-incremental

@@ -35,7 +35,8 @@ def compile_and_stat(compile_args, output_file):
     """
     subprocess.check_call(compile_args)
 
-    md5 = subprocess.check_output(["md5", "-q", output_file], text=True)
+    md5 = subprocess.check_output(["md5", "-q", output_file],
+                                  universal_newlines=True)
     mtime = time.ctime(os.path.getmtime(output_file))
 
     if VERBOSE:

utils/refactor-check-compiles.py

@@ -9,7 +9,7 @@
 
 def run_cmd(cmd, desc):
     try:
-        return subprocess.check_output(cmd)
+        return subprocess.check_output(cmd, universal_newlines=True)
     except subprocess.CalledProcessError:
         print('FAILED ' + desc + ':', file=sys.stderr)
         print(' '.join(cmd), file=sys.stderr)
@@ -130,7 +130,7 @@ def main():
         '-source-filename', args.source_filename,
         '-rewritten-output-file', temp_file_path,
         '-pos', args.pos
-    ] + extra_refactor_args + extra_both_args, desc='producing edit').decode("utf-8")
+    ] + extra_refactor_args + extra_both_args, desc='producing edit')
     sys.stdout.write(dump_text_output)
 
     run_cmd([

utils/swift-darwin-postprocess.py

@@ -32,7 +32,8 @@ def main(arguments):
 # (rdar://78851265)
 def unrpathize(filename):
     dylibsOutput = subprocess.check_output(
-        ['xcrun', 'dyldinfo', '-dylibs', filename])
+        ['xcrun', 'dyldinfo', '-dylibs', filename],
+        universal_newlines=True)
 
     # Do not rewrite @rpath-relative load commands for these libraries:
     # they are test support libraries that are never installed under
@@ -60,8 +61,7 @@ def unrpathize(filename):
 
     # Build a command to invoke install_name_tool.
     command = ['install_name_tool']
-    for binaryline in dylibsOutput.splitlines():
-        line = binaryline.decode("utf-8", "strict")
+    for line in dylibsOutput.splitlines():
         match = dylib_regex.match(line)
         if match and match.group('filename') not in allow_list:
             command.append('-change')

utils/swift-rpathize.py

@@ -55,7 +55,8 @@ def main(arguments):
 
 def rpathize(filename):
     dylibsOutput = subprocess.check_output(
-        ['xcrun', 'dyldinfo', '-dylibs', filename])
+        ['xcrun', 'dyldinfo', '-dylibs', filename],
+        universal_newlines=True)
 
     # The output from dyldinfo -dylibs is a line of header followed by one
     # install name per line, indented with spaces.
@@ -64,8 +65,7 @@ def rpathize(filename):
 
     # Build a command to invoke install_name_tool.
     command = ['install_name_tool']
-    for binaryline in dylibsOutput.splitlines():
-        line = binaryline.decode("utf-8", "strict")
+    for line in dylibsOutput.splitlines():
         match = dylib_regex.match(line)
         if match:
             command.append('-change')

utils/swift_build_support/swift_build_support/shell.py

@@ -132,12 +132,11 @@ def capture(command, stderr=None, env=None, dry_run=None, echo=True,
         _env = dict(os.environ)
         _env.update(env)
     try:
-        out = subprocess.check_output(command, env=_env, stderr=stderr)
-        # Coerce to `str` hack. not py3 `byte`, not py2 `unicode`.
-        return str(out.decode())
+        return subprocess.check_output(command, env=_env, stderr=stderr,
+                                       universal_newlines=True)
     except subprocess.CalledProcessError as e:
         if allow_non_zero_exit:
-            return str(e.output.decode())
+            return e.output
         if optional:
             return None
         _fatal_error(
@@ -213,25 +212,29 @@ def run(*args, **kwargs):
     echo_output = kwargs.pop('echo', False)
     dry_run = kwargs.pop('dry_run', False)
     env = kwargs.pop('env', None)
+    prefix = kwargs.pop('prefix', '')
     if dry_run:
-        _echo_command(dry_run, *args, env=env)
+        _echo_command(dry_run, *args, env=env, prompt="{0}+ ".format(prefix))
         return(None, 0, args)
 
     my_pipe = subprocess.Popen(
-        *args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
-    (stdout, stderr) = my_pipe.communicate()
+        *args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+        universal_newlines=True,
+        **kwargs)
+    (output, _) = my_pipe.communicate()
     ret = my_pipe.wait()
 
     if lock:
         lock.acquire()
     if echo_output:
-        print(repo_path)
-        _echo_command(dry_run, *args, env=env)
-        if stdout:
-            print(stdout, end="")
-        if stderr:
-            print(stderr, end="")
-        print()
+        sys.stdout.flush()
+        sys.stderr.flush()
+        _echo_command(dry_run, *args, env=env, prompt="{0}+ ".format(prefix))
+        if output:
+            for line in output.splitlines():
+                print("{0}{1}".format(prefix, line))
+        sys.stdout.flush()
+        sys.stderr.flush()
     if lock:
         lock.release()
 
@@ -240,6 +243,6 @@ def run(*args, **kwargs):
         eout.ret = ret
         eout.args = args
         eout.repo_path = repo_path
-        eout.stderr = stderr
+        eout.stderr = output
         raise eout
-    return (stdout, 0, args)
+    return (output, 0, args)

utils/update_checkout/update_checkout/update_checkout.py

@@ -70,7 +70,7 @@ def check_parallel_results(results, op):
             print("%s failed (ret=%d): %s" % (r.repo_path, r.ret, r))
             fail_count += 1
             if r.stderr:
-                print(r.stderr.decode('utf-8'))
+                print(r.stderr)
     return fail_count
 
 
@@ -125,7 +125,9 @@ def update_single_repository(pool_args):
         return
 
     try:
-        print("Updating '" + repo_path + "'")
+        prefix = "[{0}] ".format(os.path.basename(repo_path)).ljust(40)
+        print(prefix + "Updating '" + repo_path + "'")
+
         with shell.pushd(repo_path, dry_run=False, echo=False):
             cross_repo = False
             checkout_target = None
@@ -141,15 +143,20 @@ def update_single_repository(pool_args):
 
             # The clean option restores a repository to pristine condition.
             if should_clean:
-                shell.run(['git', 'clean', '-fdx'], echo=True)
-                shell.run(['git', 'submodule', 'foreach', '--recursive', 'git',
-                           'clean', '-fdx'], echo=True)
-                shell.run(['git', 'submodule', 'foreach', '--recursive', 'git',
-                           'reset', '--hard', 'HEAD'], echo=True)
-                shell.run(['git', 'reset', '--hard', 'HEAD'], echo=True)
+                shell.run(['git', 'clean', '-fdx'],
+                          echo=True, prefix=prefix)
+                shell.run(['git', 'submodule', 'foreach', '--recursive',
+                           'git', 'clean', '-fdx'],
+                          echo=True, prefix=prefix)
+                shell.run(['git', 'submodule', 'foreach', '--recursive',
+                           'git', 'reset', '--hard', 'HEAD'],
+                          echo=True, prefix=prefix)
+                shell.run(['git', 'reset', '--hard', 'HEAD'],
+                          echo=True, prefix=prefix)
                 # It is possible to reset --hard and still be mid-rebase.
                 try:
-                    shell.run(['git', 'rebase', '--abort'], echo=True)
+                    shell.run(['git', 'rebase', '--abort'],
+                              echo=True, prefix=prefix)
                 except Exception:
                     pass
 
@@ -166,29 +173,32 @@ def update_single_repository(pool_args):
                 except Exception:
                     shell.run(["git", "fetch", "--recurse-submodules=yes",
                                "--tags"],
-                              echo=True)
+                              echo=True, prefix=prefix)
 
                 try:
-                    shell.run(['git', 'checkout', checkout_target], echo=True)
+                    shell.run(['git', 'checkout', checkout_target],
+                              echo=True, prefix=prefix)
                 except Exception as originalException:
                     try:
                         result = shell.run(['git', 'rev-parse', checkout_target])
                         revision = result[0].strip()
-                        shell.run(['git', 'checkout', revision], echo=True)
+                        shell.run(['git', 'checkout', revision],
+                                  echo=True, prefix=prefix)
                     except Exception:
                         raise originalException
 
             # It's important that we checkout, fetch, and rebase, in order.
             # .git/FETCH_HEAD updates the not-for-merge attributes based on
             # which branch was checked out during the fetch.
             shell.run(["git", "fetch", "--recurse-submodules=yes", "--tags"],
-                      echo=True)
+                      echo=True, prefix=prefix)
 
             # If we were asked to reset to the specified branch, do the hard
             # reset and return.
             if checkout_target and reset_to_remote and not cross_repo:
                 full_target = full_target_name('origin', checkout_target)
-                shell.run(['git', 'reset', '--hard', full_target], echo=True)
+                shell.run(['git', 'reset', '--hard', full_target],
+                          echo=True, prefix=prefix)
                 return
 
             # Query whether we have a "detached HEAD", which will mean that
@@ -215,13 +225,15 @@ def update_single_repository(pool_args):
             # --rebase" that respects rebase.autostash.  See
             # http://stackoverflow.com/a/30209750/125349
             if not cross_repo and not detached_head:
-                shell.run(["git", "rebase", "FETCH_HEAD"], echo=True)
+                shell.run(["git", "rebase", "FETCH_HEAD"],
+                          echo=True, prefix=prefix)
             elif detached_head:
-                print(repo_path,
-                      "\nDetached HEAD; probably checked out a tag. No need "
-                      "to rebase.\n")
+                print(prefix +
+                      "Detached HEAD; probably checked out a tag. No need "
+                      "to rebase.")
 
-            shell.run(["git", "submodule", "update", "--recursive"], echo=True)
+            shell.run(["git", "submodule", "update", "--recursive"],
+                      echo=True, prefix=prefix)
     except Exception:
         (type, value, tb) = sys.exc_info()
         print('Error on repo "%s": %s' % (repo_path, traceback.format_exc()))
@@ -431,12 +443,12 @@ def validate_config(config):
 
 
 def full_target_name(repository, target):
-    tag = shell.capture(["git", "tag", "-l", target], echo=True).strip()
+    tag = shell.capture(["git", "tag", "-l", target], echo=False).strip()
     if tag == target:
         return tag
 
     branch = shell.capture(["git", "branch", "--list", target],
-                           echo=True).strip().replace("* ", "")
+                           echo=False).strip().replace("* ", "")
     if branch == target:
         name = "%s/%s" % (repository, target)
         return name