Merge pull request #627 from screamerbg/feature_cache_cmd

New feature: mbed cache

Author: screamerbg

README.md

@@ -851,6 +851,29 @@ You can combine the options of the Mbed update command for the following scenari
 
 Use these with caution because your uncommitted changes and unpublished libraries cannot be restored.
 
+## Repository caching
+
+To minimize traffic and reduce import times, by default Mbed CLI caches repositories by storing their indexes under the Mbed CLI user config folder - typically `~/.mbed/mbed-cache/` on UNIX systems, or `%userprofile%/.mbed/mbed-cache/` on Windows systems. Compared to a fully checked out repository, indexes are significantly smaller in size and number of files and contain the whole revision history of that repository. This allows Mbed CLI to quickly create copies of previously downloaded repository indexes and pull/fetch only the latest changes from the remote repositories, therefore dramatically reducing network traffic and download times, especially for big repositories such as `mbed-os`.
+
+You can manage the Mbed CLI caching behavior with the following subcommands:
+
+```
+mbed cache [on|off|dir <path>|ls|purge|-h|--help]
+```
+
+ * `on` - Turn repository caching on. This uses either the user specified cache directory or the default one. See "dir".
+ * `off` - Turn repository caching off. Note that this doesn't purge cached repositories. See "purge".
+ * `dir` - Set cache directory. Set to "default" to let Mbed CLI determine the cache directory location. Typically, this is `~/.mbed/mbed-cache/` on UNIX systems, or `%%userprofile%%/.mbed/mbed-cache/` on Windows systems.
+ * `ls` - List cached repositories and their size.
+ * `purge` - Purge cached repositories. Note that this doesn't turn caching off.
+ * `-h` or `--help` - Print cache command options.
+
+If no subcommand is specified to `mbed cache`, Mbed CLI prints the current cache setting (ENABLED or DISABLED) and the path to the local cache directory.
+
+For safety reasons, Mbed CLI uses the `mbed-cache` subfolder to a user specified location. This ensures that no user files are deleted during `purge` even if the user has specified root/system folder as a cache location (for example, `mbed cache dir /` or `mbed cache dir C:\`).
+
+**Security notice**: If you use cache location outside your user home/profile directory, then other system users might be able to access the repository cache and therefore the data of the cached repositories.
+
 ## Mbed CLI configuration
 
 You can streamline many options in Mbed CLI with global and local configuration.
@@ -874,7 +897,6 @@ Here is a list of configuration settings and their defaults:
  * `ARM_PATH`, `ARMC6_PATH`, `GCC_ARM_PATH`, `IAR_PATH` - defines the path to Arm Compiler 5 and 6, GCC Arm and IAR Workbench toolchains. Default: none.
  * `protocol` - defines the default protocol used for importing or cloning of programs and libraries. The possible values are `https`, `http` and `ssh`. Use `ssh` if you have generated and registered SSH keys (Public Key Authentication) with a service such as GitHub, GitLab, Bitbucket and so on. Read more about SSH keys [here](https://help.github.com/articles/generating-an-ssh-key/). Default: `https`.
  * `depth` - defines the *clone* depth for importing or cloning and applies only to *Git* repositories. Note that though this option may improve cloning speed, it may also prevent you from correctly checking out a dependency tree when the reference revision hash is older than the clone depth. Read more about shallow clones [here](https://git-scm.com/docs/git-clone). Default: none.
- * `cache` - defines the local path that stores small copies of the imported or cloned repositories, and Mbed CLI uses it to minimize traffic and speed up future imports of the same repositories. Use `on` or `enabled` to turn on caching in the system temp path. Use `none` to turn caching off. Default: none (disabled).
 
 ## Troubleshooting
 

mbed/mbed.py

@@ -255,6 +255,12 @@ def remove_readonly(func, path, _):
             func(path)
         shutil.rmtree(directory, onerror=remove_readonly)
 
+def sizeof_fmt(num, suffix='B'):
+    for unit in ['','K','M','G','T','P','E','Z']:
+        if abs(num) < 1024.0:
+            return "%3.1f%s%s" % (num, unit, suffix)
+        num /= 1024.0
+    return "%.1f%s%s" % (num, 'Yi', suffix)
 
 # Directory navigation
 @contextlib.contextmanager
@@ -995,10 +1001,9 @@ def fromurl(cls, url, path=None):
         else:
             error('Invalid repository (%s)' % url.strip(), -1)
 
-        cache_cfg = Global().get_cfg('CACHE', '')
-        if cache_repositories and cache_cfg and cache_cfg != 'none' and cache_cfg != 'off' and cache_cfg != 'disabled':
-            loc = cache_cfg if (cache_cfg and cache_cfg != 'on' and cache_cfg != 'enabled') else None
-            repo.cache = loc or os.path.join(tempfile.gettempdir(), 'mbed-repo-cache')
+        cache_cfg = Global().cache_cfg()
+        if cache_repositories and cache_cfg['cache'] == 'enabled':
+            repo.cache = cache_cfg['cache_dir']
 
         return repo
 
@@ -1030,10 +1035,9 @@ def fromrepo(cls, path=None):
         repo.path = os.path.abspath(path)
         repo.name = os.path.basename(repo.path)
 
-        cache_cfg = Global().get_cfg('CACHE', '')
-        if cache_repositories and cache_cfg and cache_cfg != 'none' and cache_cfg != 'off' and cache_cfg != 'disabled':
-            loc = cache_cfg if (cache_cfg and cache_cfg != 'on' and cache_cfg != 'enabled') else None
-            repo.cache = loc or os.path.join(tempfile.gettempdir(), 'mbed-repo-cache')
+        cache_cfg = Global().cache_cfg()
+        if cache_repositories and cache_cfg['cache'] == 'enabled':
+            repo.cache = cache_cfg['cache_dir']
 
         repo.sync()
 
@@ -1638,6 +1642,10 @@ def set_cfg(self, *args, **kwargs):
     def list_cfg(self, *args, **kwargs):
         return Cfg(self.path).list(*args, **kwargs)
 
+    def cache_cfg(self, *args, **kwargs):
+        return Cfg(self.path).cache(*args, **kwargs)
+
+
 # Cfg classed used for handling the config backend
 class Cfg(object):
     path = None
@@ -1703,6 +1711,16 @@ def list(self):
                 vars[m.group(1)] = m.group(2)
         return vars
 
+    # Get cache configuration
+    def cache(self):
+        cache_cfg = self.get('CACHE', 'enabled')
+        cache_val = 'enabled' if cache_repositories and cache_cfg and cache_cfg != 'none' and cache_cfg != 'off' and cache_cfg != 'disabled' else 'disabled'
+
+        cache_dir_cfg = self.get('CACHE_DIR', None)
+        loc = cache_dir_cfg if cache_dir_cfg != 'default' else (cache_cfg if (cache_cfg and cache_cfg != 'on' and cache_cfg != 'off' and cache_cfg != 'none' and cache_cfg != 'enabled' and cache_cfg != 'disabled') else None)
+        cache_base = loc or Global().path
+        return {'cache': cache_val, 'cache_base': cache_base, 'cache_dir': os.path.join(cache_base, 'mbed-cache')}
+
 
 def formaturl(url, format="default"):
     url = "%s" % url
@@ -2748,11 +2766,12 @@ def target_(name=None, global_cfg=False, supported=False):
         return compile_(supported=supported)
     return config_('target', name, global_cfg=global_cfg)
 
+
 @subcommand('toolchain',
     dict(name='name', nargs='?', help='Default toolchain name. Example: ARM, GCC_ARM, IAR'),
     dict(name=['-G', '--global'], dest='global_cfg', action='store_true', help='Use global settings, not local'),
     dict(name=['-S', '--supported'], dest='supported', action='store_true', help='Shows supported matrix of targets and toolchains'),
-    help='Set or get default toolchain\n\n',
+    help='Set or get default toolchain',
     description=(
         "Set or get default toolchain\n"
         "This is an alias to 'mbed config [--global] toolchain [name]'\n"))
@@ -2761,6 +2780,78 @@ def toolchain_(name=None, global_cfg=False, supported=False):
         return compile_(supported=supported)
     return config_('toolchain', name, global_cfg=global_cfg)
 
+
+@subcommand('cache',
+    dict(name='on', nargs='?', help='Turn repository caching on. Will use either the default or the user specified cache directory.'),
+    dict(name='off', nargs='?', help='Turn repository caching off. Note that this doesn\'t purge cached repositories. See "purge".'),
+    dict(name='dir', nargs='?', help='Set cache directory. Set to "default" to let mbed CLI determine the cache directory location (%s/mbed-cache/).' % Global().path),
+    dict(name='ls', nargs='?', help='List cached repositories and their sizes.'),
+    dict(name='purge', nargs='?', help='Purge cached repositories. Note that this doesn\'t turn caching off'),
+    help='Repository cache management\n\n',
+    description=(
+        "Repository cache management\n"
+        "To minimize traffic and reduce import times, Mbed CLI can cache repositories by storing their indexes.\n"
+        "By default repository caching is turned on. Turn it off if you experience any problems.\n"))
+def cache_(on=False, off=False, dir=None, ls=False, purge=False, global_cfg=False):
+    cmd = str(on).lower()
+    argument = off
+    g = Global()
+
+    cfg = g.cache_cfg()
+    if cmd == 'off' or cmd == 'on':
+        g.set_cfg('CACHE', 'enabled' if cmd == 'on' else 'disabled')
+        cfg = g.cache_cfg()
+        action("Repository cache is now %s." % str(cfg['cache']).upper())
+        action("Cache location \"%s\"" % cfg['cache_dir'])
+    elif cmd == 'dir':
+        if not argument:
+            error("Please specify directory or path to cache repositories. Alternatively specify \"default\" to cache repositories in the default user home location.")
+        if not os.path.exists(argument):
+            try:
+                os.makedirs(argument)
+            except (IOError, ImportError, OSError):
+                error("Unable to create cache directory \"%s\"" % argument, 128)
+        elif not os.path.isdir(argument):
+            error("The specified location \"%s\" is not a directory" % argument, 128)
+        elif len(os.listdir(argument)) > 1:
+            warning("Directory \"%s\" is not empty." % argument)
+        g.set_cfg('CACHE_DIR', argument)
+        action('Repository cache location set to \"%s\"' % argument)
+    elif cmd == 'ls':
+        def get_size_(path):
+            size = 0
+            for dirpath, dirs, files in os.walk(path):
+                for f in files:
+                    size += os.path.getsize(os.path.join(dirpath, f))
+            return size
+        action("Listing cached repositories in \"%s\"" % cfg['cache_base'])
+        repos = []
+        total_size = 0
+        for dirpath, dirs, files in os.walk(cfg['cache_dir']):
+            dirs[:] = [d for d in dirs  if not d.startswith('.')]
+            if Repo.isrepo(dirpath):
+                repo = Repo().fromrepo(dirpath)
+                url = repo.url
+                size = get_size_(repo.path)
+                total_size += size
+                log("* %s %s\n" % ('{:68}'.format(url), sizeof_fmt(size).rjust(8)))
+                for d in dirs:
+                    dirs.remove(d)
+        log(("-" * 79) + "\n")
+        log("%s %s\n" % ('{:70}'.format('Total size:'), sizeof_fmt(total_size).rjust(8)))
+    elif cmd == 'purge':
+        action("Purging cached repositories in \"%s\"..." % cfg['cache_base'])
+        if os.path.isdir(cfg['cache_dir']):
+            rmtree_readonly(cfg['cache_dir'])
+        action("Purge complete!")
+    elif cmd == "false":
+        action("Repository cache is %s." % str(cfg['cache']).upper())
+        action("Cache location \"%s\"" % cfg['cache_dir'])
+    else:
+        print cmd
+        error("Invalid cache command. Please see \"mbed cache --help\" for valid commands.")
+
+
 @subcommand('help',
     help='This help screen')
 def help_():