Ensure that the prefix name in s3_trial always includes a trailing '/' (aws#246)

* ensure prefix name in s3 trial is a dir

* refactored list_s3_objects

* run tests

Author: NihalHarish

tornasole/core/s3_utils.py

@@ -13,13 +13,15 @@ def _list_s3_prefixes(list_info):
     return files
 
 
-def list_s3_objects(bucket, prefix, start_after_key=None):
+def list_s3_objects(bucket, prefix, start_after_key=None, delimiter=""):
     last_token = None
     if start_after_key is None:
         start_after_key = prefix
     logger.debug(f'Trying to load index files after {start_after_key}')
-    list_params = {'Bucket': bucket, 'Prefix': prefix, 'StartAfter': start_after_key}
-    req = ListRequest(**list_params)
+    req = ListRequest(Bucket=bucket,
+                      Prefix=prefix,
+                      StartAfter=start_after_key,
+                      Delimiter=delimiter)
     objects = _list_s3_prefixes([req])
     if len(objects) > 0:
         last_token = objects[-1]

tornasole/trials/s3_trial.py

@@ -32,7 +32,7 @@ def __init__(self, name, bucket_name, prefix_name,
                          parallel=False, check=check, index_mode=index_mode, cache=cache)
         self.logger.info(f'Loading trial {name} at path s3://{bucket_name}/{prefix_name}')
         self.bucket_name = bucket_name
-        self.prefix_name = prefix_name
+        self.prefix_name = os.path.join(prefix_name, '')
         self.path = "s3://"+os.path.join(self.bucket_name, self.prefix_name)
         self.s3_handler = S3Handler()
         self._load_collections()