Skip to content

Use bytes or %r to handle paths and process output #766

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

Use bytes or %r to handle paths and process output #766

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8c93bdc
Use bytes for process output and CWD
theotherjimmy Oct 3, 2018
6c9de68
Use bytes for newline char
theotherjimmy Oct 3, 2018
9a0ffa8
Write bytes to stdout and stderr corretly on python3
theotherjimmy Oct 3, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 44 additions & 40 deletions mbed/mbed.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,10 @@

# Logging and output
def log(msg, is_error=False):
sys.stderr.write(msg) if is_error else sys.stdout.write(msg)
if sys.version_info[0] > 2 and isinstance(msg, bytes):
sys.stderr.buffer.write(msg) if is_error else sys.stdout.buffer.write(msg)
else:
sys.stderr.write(msg) if is_error else sys.stdout.write(msg)

def message(msg):
if very_verbose:
Expand Down Expand Up @@ -228,14 +231,14 @@ class ProcessException(Exception):

def popen(command, **kwargs):
# print for debugging
info("Exec \"%s\" in \"%s\"" % (' '.join(command), getcwd()))
info("Exec \"%s\" in %r" % (' '.join(command), getcwd()))
proc = None
try:
proc = subprocess.Popen(command, **kwargs)
except OSError as e:
if e.args[0] == errno.ENOENT:
error(
"Could not execute \"%s\" in \"%s\".\n"
"Could not execute \"%s\" in %r.\n"
"You can verify that it's installed and accessible from your current path by executing \"%s\".\n" % (' '.join(command), getcwd(), command[0]), e.args[0])
else:
raise e
Expand All @@ -246,13 +249,13 @@ def popen(command, **kwargs):

def pquery(command, output_callback=None, stdin=None, **kwargs):
if very_verbose:
info("Exec \"%s\" in \"%s\"" % (' '.join(command), getcwd()))
info("Exec \"%s\" in %r" % (' '.join(command), getcwd()))
try:
proc = subprocess.Popen(command, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
except OSError as e:
if e.args[0] == errno.ENOENT:
error(
"Could not execute \"%s\" in \"%s\".\n"
"Could not execute \"%s\" in %r.\n"
"You can verify that it's installed and accessible from your current path by executing \"%s\".\n" % (' '.join(command), getcwd(), command[0]), e.args[0])
else:
raise e
Expand All @@ -274,12 +277,12 @@ def pquery(command, output_callback=None, stdin=None, **kwargs):
stdout, _ = proc.communicate(stdin)

if very_verbose:
log(stdout.decode("utf-8").strip() + "\n")
log(stdout.strip() + b'\n')

if proc.returncode != 0:
raise ProcessException(proc.returncode, command[0], ' '.join(command), getcwd())

return stdout.decode("utf-8")
return stdout

def rmtree_readonly(directory):
if os.path.islink(directory):
Expand Down Expand Up @@ -372,6 +375,7 @@ def clone(url, path=None, depth=None, protocol=None):
except Exception as e:
if os.path.isdir(path):
rmtree_readonly(path)
raise
error(e.args[1], e.args[0])

def fetch_rev(url, rev):
Expand All @@ -392,12 +396,12 @@ def unpack_rev(rev):
rev_file = os.path.join('.'+Bld.name, '.rev-' + rev + '.zip')
try:
with zipfile.ZipFile(rev_file) as zf:
action("Unpacking library build \"%s\" in \"%s\"" % (rev, getcwd()))
action("Unpacking library build \"%s\" in %r" % (rev, getcwd()))
zf.extractall('.')
except:
if os.path.isfile(rev_file):
os.remove(rev_file)
raise Exception(128, "An error occurred while unpacking library archive \"%s\" in \"%s\"" % (rev_file, getcwd()))
raise Exception(128, "An error occurred while unpacking library archive \"%s\" in %r" % (rev_file, getcwd()))

def checkout(rev, clean=False):
url = Bld.geturl()
Expand All @@ -414,7 +418,7 @@ def checkout(rev, clean=False):
if rev != Bld.getrev() or clean:
Bld.cleanup()

info("Checkout \"%s\" in %s" % (rev, os.path.basename(getcwd())))
info("Checkout \"%s\" in %r" % (rev, os.path.basename(getcwd())))
try:
Bld.unpack_rev(rev)
Bld.seturl(url+'/'+rev)
Expand All @@ -431,7 +435,7 @@ def isvalidurl(url):
return re.match(regex_build_url, url.strip().replace('\\', '/'))

def seturl(url):
info("Setting url to \"%s\" in %s" % (url, getcwd()))
info("Setting url to \"%s\" in %r" % (url, getcwd()))
if not os.path.exists('.'+Bld.name):
os.mkdir('.'+Bld.name)

Expand Down Expand Up @@ -504,19 +508,19 @@ def publish(all_refs=None):
popen([hg_cmd, 'push'] + (['--new-branch'] if all_refs else []) + (['-v'] if very_verbose else ([] if verbose else ['-q'])))

def fetch():
info("Fetching revisions from remote repository to \"%s\"" % os.path.basename(getcwd()))
info("Fetching revisions from remote repository to %r" % os.path.basename(getcwd()))
popen([hg_cmd, 'pull'] + (['-v'] if very_verbose else ([] if verbose else ['-q'])))

def discard():
info("Discarding local changes in \"%s\"" % os.path.basename(getcwd()))
info("Discarding local changes in %r" % os.path.basename(getcwd()))
popen([hg_cmd, 'update', '-C'] + (['-v'] if very_verbose else ([] if verbose else ['-q'])))

def checkout(rev, clean=False, clean_files=False):
info("Checkout \"%s\" in %s" % (rev if rev else "latest", os.path.basename(getcwd())))
info("Checkout \"%s\" in %r" % (rev if rev else "latest", os.path.basename(getcwd())))
if clean_files:
files = pquery([hg_cmd, 'status', '--no-status', '-ui']).splitlines()
for f in files:
info("Remove untracked file \"%s\"" % f)
info("Remove untracked file %r" % f)
os.remove(f)
popen([hg_cmd, 'update'] + (['-C'] if clean else []) + (['-r', rev] if rev else []) + (['-v'] if very_verbose else ([] if verbose else ['-q'])))

Expand Down Expand Up @@ -544,7 +548,7 @@ def outgoing():
return 0

def seturl(url):
info("Setting url to \"%s\" in %s" % (url, getcwd()))
info("Setting url to \"%s\" in %r" % (url, getcwd()))
hgrc = os.path.join('.hg', 'hgrc')
tagpaths = '[paths]'
remote = 'default'
Expand Down Expand Up @@ -589,7 +593,7 @@ def geturl():
if default_url:
url = default_url

return formaturl(url or pquery([hg_cmd, 'paths', 'default']).strip())
return formaturl(url or pquery([hg_cmd, 'paths', 'default']).strip().decode("utf-8"))

def getrev():
if os.path.isfile(os.path.join('.hg', 'dirstate')):
Expand All @@ -600,19 +604,19 @@ def getrev():
return ""

def getbranch():
return pquery([hg_cmd, 'branch']).strip() or ""
return pquery([hg_cmd, 'branch']).strip().decode("utf-8") or ""

def gettags():
tags = []
refs = pquery([hg_cmd, 'tags']).strip().splitlines() or []
refs = pquery([hg_cmd, 'tags']).strip().decode("utf-8").splitlines() or []
for ref in refs:
m = re.match(r'^(.+?)\s+(\d+)\:([a-f0-9]+)$', ref)
if m:
tags.append([m.group(3), m.group(1)])
return tags

def remoteid(url, rev=None):
return pquery([hg_cmd, 'id', '--id', url] + (['-r', rev] if rev else [])).strip() or ""
return pquery([hg_cmd, 'id', '--id', url] + (['-r', rev] if rev else [])).strip().decode("utf-8") or ""

def hgrc():
hook = 'ignore.local = .hg/hgignore'
Expand All @@ -637,7 +641,7 @@ def ignores():
with open(Hg.ignore_file, 'w') as f:
f.write("syntax: glob\n"+'\n'.join(ignores)+'\n')
except IOError:
error("Unable to write ignore file in \"%s\"" % os.path.join(getcwd(), Hg.ignore_file), 1)
error("Unable to write ignore file in %r" % os.path.join(getcwd(), Hg.ignore_file), 1)

def ignore(dest):
Hg.hgrc()
Expand All @@ -652,7 +656,7 @@ def ignore(dest):
with open(Hg.ignore_file, 'a') as f:
f.write(dest + '\n')
except IOError:
error("Unable to write ignore file in \"%s\"" % os.path.join(getcwd(), Hg.ignore_file), 1)
error("Unable to write ignore file in %r" % os.path.join(getcwd(), Hg.ignore_file), 1)

def unignore(dest):
Hg.ignore_file = os.path.join('.hg', 'hgignore')
Expand All @@ -668,7 +672,7 @@ def unignore(dest):
with open(Hg.ignore_file, 'w') as f:
f.write('\n'.join(lines) + '\n')
except IOError:
error("Unable to write ignore file in \"%s\"" % os.path.join(getcwd(), Hg.ignore_file), 1)
error("Unable to write ignore file in %r" % os.path.join(getcwd(), Hg.ignore_file), 1)

def action_progress(line, sep):
m = re.match(r'(\w+).+?\s+(\d+)/(\d+)\s+.*?', line)
Expand All @@ -694,11 +698,11 @@ def cleanup():
info("Cleaning up Git index")
pquery([git_cmd, 'checkout', '--detach', 'HEAD'] + ([] if very_verbose else ['-q'])) # detach head so local branches are deletable
branches = []
lines = pquery([git_cmd, 'branch']).strip().splitlines() # fetch all local branches
lines = pquery([git_cmd, 'branch']).strip().decode("utf-8").splitlines() # fetch all local branches
for line in lines:
if re.match(r'^\*?\s+\((.+)\)$', line):
if re.match(b'^\\*?\\s+\\((.+)\\)$', line):
continue
line = re.sub(r'\s+', '', line)
line = re.sub(b'\\s+', '', line)
branches.append(line)

for branch in branches: # delete all local branches so the new repo clone is not poluted
Expand Down Expand Up @@ -737,30 +741,30 @@ def publish(all_refs=None):
if remote and branch:
popen([git_cmd, 'push', remote, branch] + (['-v'] if very_verbose else ([] if verbose else ['-q'])))
else:
err = "Unable to publish outgoing changes for \"%s\" in \"%s\".\n" % (os.path.basename(getcwd()), getcwd())
err = "Unable to publish outgoing changes for %r in %r.\n" % (os.path.basename(getcwd()), getcwd())
if not remote:
error(err+"The local repository is not associated with a remote one.", 1)
if not branch:
error(err+"Working set is not on a branch.", 1)

def fetch():
info("Fetching revisions from remote repository to \"%s\"" % os.path.basename(getcwd()))
info("Fetching revisions from remote repository to %r" % os.path.basename(getcwd()))
popen([git_cmd, 'fetch', '--all', '--tags'] + (['-v'] if very_verbose else ([] if verbose else ['-q'])))

def discard(clean_files=False):
info("Discarding local changes in \"%s\"" % os.path.basename(getcwd()))
info("Discarding local changes in %r" % os.path.basename(getcwd()))
pquery([git_cmd, 'reset', 'HEAD'] + ([] if very_verbose else ['-q'])) # unmarks files for commit
pquery([git_cmd, 'checkout', '.'] + ([] if very_verbose else ['-q'])) # undo modified files
pquery([git_cmd, 'clean', '-fd'] + (['-x'] if clean_files else []) + (['-q'] if very_verbose else ['-q'])) # cleans up untracked files and folders

def merge(dest):
info("Merging \"%s\" with \"%s\"" % (os.path.basename(getcwd()), dest))
info("Merging \"%s\" with %r" % (os.path.basename(getcwd()), dest))
popen([git_cmd, 'merge', dest] + (['-v'] if very_verbose else ([] if verbose else ['-q'])))

def checkout(rev, clean=False):
if not rev:
return
info("Checkout \"%s\" in %s" % (rev, os.path.basename(getcwd())))
info("Checkout \"%s\" in %r" % (rev, os.path.basename(getcwd())))
branch = None
refs = Git.getbranches(rev)
for ref in refs: # re-associate with a local or remote branch (rev is the same)
Expand Down Expand Up @@ -795,7 +799,7 @@ def update(rev=None, clean=False, clean_files=False, is_local=False):
except ProcessException:
pass
else:
err = "Unable to update \"%s\" in \"%s\"." % (os.path.basename(getcwd()), getcwd())
err = "Unable to update %r in %r." % (os.path.basename(getcwd()), getcwd())
if not remote:
info(err+"\nThe local repository is not associated with a remote one.\nYou should associate your repository with a remote one.")
if not branch:
Expand All @@ -808,7 +812,7 @@ def dirty():
return pquery([git_cmd, 'status', '-uno', '--porcelain'])

def untracked():
return pquery([git_cmd, 'ls-files', '--others', '--exclude-standard']).splitlines()
return pquery([git_cmd, 'ls-files', '--others', '--exclude-standard']).decode("utf-8").splitlines()

def outgoing():
# Get default remote
Expand Down Expand Up @@ -861,7 +865,7 @@ def getremotes(rtype='fetch'):
return result

def seturl(url):
info("Setting url to \"%s\" in %s" % (url, getcwd()))
info("Setting url to \"%s\" in %r" % (url, getcwd()))
return pquery([git_cmd, 'remote', 'set-url', 'origin', url]).strip()

def geturl():
Expand Down Expand Up @@ -937,7 +941,7 @@ def ignores():
with open(Git.ignore_file, 'w') as f:
f.write('\n'.join(ignores)+'\n')
except IOError:
error("Unable to write ignore file in \"%s\"" % os.path.join(getcwd(), Git.ignore_file), 1)
error("Unable to write ignore file in %r" % os.path.join(getcwd(), Git.ignore_file), 1)

def ignore(dest):
try:
Expand All @@ -955,7 +959,7 @@ def ignore(dest):
with open(Git.ignore_file, 'a') as f:
f.write(dest.replace("\\", "/") + '\n')
except IOError:
error("Unable to write ignore file in \"%s\"" % os.path.join(getcwd(), Git.ignore_file), 1)
error("Unable to write ignore file in %r" % os.path.join(getcwd(), Git.ignore_file), 1)
def unignore(dest):
try:
with open(Git.ignore_file) as f:
Expand All @@ -973,7 +977,7 @@ def unignore(dest):
with open(Git.ignore_file, 'w') as f:
f.write('\n'.join(lines) + '\n')
except IOError:
error("Unable to write ignore file in \"%s\"" % os.path.join(getcwd(), Git.ignore_file), 1)
error("Unable to write ignore file in %r" % os.path.join(getcwd(), Git.ignore_file), 1)

def action_progress(line, sep):
m = re.match(r'([\w :]+)\:\s*(\d+)% \((\d+)/(\d+)\)', line)
Expand Down Expand Up @@ -1056,7 +1060,7 @@ def fromrepo(cls, path=None):
path = Repo.findparent(getcwd())
if path is None:
error(
"Could not find mbed program in current path \"%s\".\n"
"Could not find mbed program in current path %r.\n"
"You can fix this by calling \"mbed new .\" or \"mbed config root .\" in the root of your program." % getcwd())

repo.path = os.path.abspath(path)
Expand Down Expand Up @@ -3153,7 +3157,7 @@ def config_(var=None, value=None, global_cfg=False, unset=False, list_config=Fal
log("\n")

p = Program(getcwd())
action("Local config (%s):" % p.path)
action("Local config (%r):" % p.path)
if not p.is_cwd:
p_vars = p.list_cfg().items()
if p_vars:
Expand Down Expand Up @@ -3330,7 +3334,7 @@ def main():
try:
very_verbose = pargs.very_verbose
verbose = very_verbose or pargs.verbose
info('Working path \"%s\" (%s)' % (getcwd(), Repo.pathtype(cwd_root)))
info('Working path %r (%s)' % (getcwd(), Repo.pathtype(cwd_root)))
status = pargs.command(pargs)
except ProcessException as e:
error(
Expand Down