python black --line-length=100 (aws#279)

Author: jarednielsen

bin/build_binaries.py

@@ -4,47 +4,71 @@
 import os
 import boto3
 
-parser = argparse.ArgumentParser(description='Build Tornasole binaries')
-parser.add_argument('--upload', default=False,
-                    dest='upload', action='store_true',
-                    help='Pass --upload if you want to upload the binaries'
-                         'built to the s3 location')
-parser.add_argument('--replace-latest', default=False,
-                    dest='replace_latest', action='store_true',
-                    help='Pass --replace-latest if you want to upload the new binary to '
-                         'replace the latest binary in the S3 location. Note that'
-                         'this also requires you to pass --upload')
+parser = argparse.ArgumentParser(description="Build Tornasole binaries")
+parser.add_argument(
+    "--upload",
+    default=False,
+    dest="upload",
+    action="store_true",
+    help="Pass --upload if you want to upload the binaries" "built to the s3 location",
+)
+parser.add_argument(
+    "--replace-latest",
+    default=False,
+    dest="replace_latest",
+    action="store_true",
+    help="Pass --replace-latest if you want to upload the new binary to "
+    "replace the latest binary in the S3 location. Note that"
+    "this also requires you to pass --upload",
+)
 args = parser.parse_args()
 exec(open("tornasole/_version.py").read())
 
 VERSION = __version__
-BINARIES = [
-    'mxnet',
-    'tensorflow',
-    'pytorch',
-    'xgboost',
-    'rules'
-]
+BINARIES = ["mxnet", "tensorflow", "pytorch", "xgboost", "rules"]
 
 for b in BINARIES:
-    if b == 'rules':
-        env_var = 'TORNASOLE_FOR_RULES'
+    if b == "rules":
+        env_var = "TORNASOLE_FOR_RULES"
     else:
-        env_var = 'TORNASOLE_WITH_' + b.upper()
+        env_var = "TORNASOLE_WITH_" + b.upper()
     env = dict(os.environ)
-    env[env_var] = '1'
-    subprocess.check_call([sys.executable, 'setup.py', 'bdist_wheel', '--universal'],
-                          env=env)
+    env[env_var] = "1"
+    subprocess.check_call([sys.executable, "setup.py", "bdist_wheel", "--universal"], env=env)
     if args.upload:
-        subprocess.check_call(['aws', 's3', 'cp', 'dist/tornasole-{}-py2.py3-none-any.whl'.format(VERSION),
-                               's3://tornasole-binaries-use1/tornasole_{}/py3/'.format(b)])
+        subprocess.check_call(
+            [
+                "aws",
+                "s3",
+                "cp",
+                "dist/tornasole-{}-py2.py3-none-any.whl".format(VERSION),
+                "s3://tornasole-binaries-use1/tornasole_{}/py3/".format(b),
+            ]
+        )
 
         if args.replace_latest:
             # upload current version
-            subprocess.check_call(['aws', 's3', 'cp',
-                                   's3://tornasole-binaries-use1/tornasole_{}/py3/tornasole-{}-py2.py3-none-any.whl'.format(b, VERSION),
-                                   's3://tornasole-binaries-use1/tornasole_{}/py3/latest/'.format(b)])
+            subprocess.check_call(
+                [
+                    "aws",
+                    "s3",
+                    "cp",
+                    "s3://tornasole-binaries-use1/tornasole_{}/py3/tornasole-{}-py2.py3-none-any.whl".format(
+                        b, VERSION
+                    ),
+                    "s3://tornasole-binaries-use1/tornasole_{}/py3/latest/".format(b),
+                ]
+            )
             # remove other versions
-            subprocess.check_call(['aws', 's3', 'rm', '--recursive', '--exclude', 'tornasole-{}*'.format(VERSION),
-                                   's3://tornasole-binaries-use1/tornasole_{}/py3/latest/'.format(b)])
-    subprocess.check_call(['rm', '-rf', 'dist', 'build', '*.egg-info', '.eggs'])
+            subprocess.check_call(
+                [
+                    "aws",
+                    "s3",
+                    "rm",
+                    "--recursive",
+                    "--exclude",
+                    "tornasole-{}*".format(VERSION),
+                    "s3://tornasole-binaries-use1/tornasole_{}/py3/latest/".format(b),
+                ]
+            )
+    subprocess.check_call(["rm", "-rf", "dist", "build", "*.egg-info", ".eggs"])

bin/build_containers.py

@@ -6,35 +6,42 @@
 
 
 FRAMEWORK_VERSIONS = {
-    'mxnet': '1.4.1',
-    'pytorch': '1.1.0',
-    'tensorflow': '1.14.0',
-    'xgboost': '0.90'
-    }
+    "mxnet": "1.4.1",
+    "pytorch": "1.1.0",
+    "tensorflow": "1.14.0",
+    "xgboost": "0.90",
+}
 
 
 def run_command(command_list, stdout, stderr):
-  subprocess.check_call(command_list, stdout=stdout, stderr=stderr)
+    subprocess.check_call(command_list, stdout=stdout, stderr=stderr)
 
 
 # you can clean all caches used by docker with `docker system prune -a`
 def build_container(framework, version, args):
-    command = ['bash', 'bin/sagemaker-containers/{}/{}/build.sh'.format(framework, version), args.tag]
+    command = [
+        "bash",
+        "bin/sagemaker-containers/{}/{}/build.sh".format(framework, version),
+        args.tag,
+    ]
     if not args.single_process:
-        with open(os.path.join(args.logs_path, '{}.log'.format(framework)), "w") as logfile:
+        with open(os.path.join(args.logs_path, "{}.log".format(framework)), "w") as logfile:
             run_command(command, stdout=logfile, stderr=logfile)
     else:
         run_command(command, None, None)
 
 
-parser = argparse.ArgumentParser(description='Build Tornasole binaries')
-parser.add_argument('--tag', type=str, default='temp',
-                    help='Pass the tag to upload the image to ECR with. '
-                         'You might want to set the tag to latest for '
-                         'final images.')
-parser.add_argument('--single-process', action='store_true',
-                    dest='single_process', default=False)
-parser.add_argument('--logs-path', type=str, default='bin/sagemaker-containers/logs/')
+parser = argparse.ArgumentParser(description="Build Tornasole binaries")
+parser.add_argument(
+    "--tag",
+    type=str,
+    default="temp",
+    help="Pass the tag to upload the image to ECR with. "
+    "You might want to set the tag to latest for "
+    "final images.",
+)
+parser.add_argument("--single-process", action="store_true", dest="single_process", default=False)
+parser.add_argument("--logs-path", type=str, default="bin/sagemaker-containers/logs/")
 args = parser.parse_args()
 
 if not os.path.exists(args.logs_path):
@@ -44,8 +51,11 @@ def build_container(framework, version, args):
 for f, v in FRAMEWORK_VERSIONS.items():
     p = Process(name=f, target=build_container, args=(f, v, args))
     p.start()
-    print('Started building container for {}. You can find the log at {}.log'
-          .format(f, os.path.join(args.logs_path, f)))
+    print(
+        "Started building container for {}. You can find the log at {}.log".format(
+            f, os.path.join(args.logs_path, f)
+        )
+    )
     if args.single_process:
         p.join()
     else:
@@ -59,6 +69,6 @@ def build_container(framework, version, args):
         for p in processes:
             if p not in ended_processes and not p.is_alive():
                 p.join()
-                print(f'Finished process {p.name}')
+                print(f"Finished process {p.name}")
                 ended_processes.add(p)
         sleep(3)

bin/sagemaker-containers/tensorflow/run_sagemaker.py

@@ -3,33 +3,39 @@
 import os
 
 dir_path = os.path.dirname(os.path.realpath(__file__))
-train_script_path = os.path.join(dir_path, 'tf-train.py')
+train_script_path = os.path.join(dir_path, "tf-train.py")
 
-tag = os.environ.get('SM_TESTING_TAG', 'DEFAULTTAGWHICHWILLFAIL')
-available_versions = ['1.13.1']
+tag = os.environ.get("SM_TESTING_TAG", "DEFAULTTAGWHICHWILLFAIL")
+available_versions = ["1.13.1"]
 
 parser = argparse.ArgumentParser()
-parser.add_argument("--tf_version", default='1.13.1', help=f"one of {available_versions}")
+parser.add_argument("--tf_version", default="1.13.1", help=f"one of {available_versions}")
 args = parser.parse_args()
 version = args.tf_version
-assert version in ['1.13.1'], f"version={version} not in {available_versions}"
+assert version in ["1.13.1"], f"version={version} not in {available_versions}"
 
-estimator = TensorFlow(entry_point=train_script_path,
-                       image_name=f'072677473360.dkr.ecr.us-east-1.amazonaws.com/tornasole-preprod-tf-{version}-cpu:' + tag,
-                       role='AmazonSageMaker-ExecutionRole-20190614T145575', # hardcode role name
-                       base_job_name='tornasole', #there are some restrictions on base job name so keep it simple
-                       train_instance_count=1,
-                       py_version='py3',
-                       framework_version=version,
-                       train_instance_type='ml.m4.xlarge')
+estimator = TensorFlow(
+    entry_point=train_script_path,
+    image_name=f"072677473360.dkr.ecr.us-east-1.amazonaws.com/tornasole-preprod-tf-{version}-cpu:"
+    + tag,
+    role="AmazonSageMaker-ExecutionRole-20190614T145575",  # hardcode role name
+    base_job_name="tornasole",  # there are some restrictions on base job name so keep it simple
+    train_instance_count=1,
+    py_version="py3",
+    framework_version=version,
+    train_instance_type="ml.m4.xlarge",
+)
 estimator.fit()
 
-estimator = TensorFlow(entry_point=train_script_path,
-                       image_name=f'072677473360.dkr.ecr.us-east-1.amazonaws.com/tornasole-preprod-tf-{version}-gpu:' + tag,
-                       role='AmazonSageMaker-ExecutionRole-20190614T145575', # hardcode role name
-                       base_job_name='tornasole', #there are some restrictions on base job name so keep it simple
-                       train_instance_count=1,
-                       py_version='py3',
-                       framework_version=version,
-                       train_instance_type='ml.p2.xlarge')
+estimator = TensorFlow(
+    entry_point=train_script_path,
+    image_name=f"072677473360.dkr.ecr.us-east-1.amazonaws.com/tornasole-preprod-tf-{version}-gpu:"
+    + tag,
+    role="AmazonSageMaker-ExecutionRole-20190614T145575",  # hardcode role name
+    base_job_name="tornasole",  # there are some restrictions on base job name so keep it simple
+    train_instance_count=1,
+    py_version="py3",
+    framework_version=version,
+    train_instance_type="ml.p2.xlarge",
+)
 estimator.fit()

bin/sagemaker-containers/tensorflow/tf-train.py

@@ -5,46 +5,49 @@
 import time
 import uuid
 from tornasole.tensorflow import TornasoleHook, TornasoleOptimizer, SaveConfig
+
 parser = argparse.ArgumentParser()
-parser.add_argument('--lr', type=float, help="Learning Rate", default=0.001 )
-parser.add_argument('--steps', type=int, help="Number of steps to run", default=100 )
-parser.add_argument('--scale', type=float, help="Scaling factor for inputs", default=1.0 )
-parser.add_argument('--tornasole_frequency', type=float, help="How often to save TS data", default=10 )
-parser.add_argument('--run_name', type=str, help="Run Name", default=str(uuid.uuid4()) )
-parser.add_argument('--local_reductions', nargs='+', type=str, default=[] )
+parser.add_argument("--lr", type=float, help="Learning Rate", default=0.001)
+parser.add_argument("--steps", type=int, help="Number of steps to run", default=100)
+parser.add_argument("--scale", type=float, help="Scaling factor for inputs", default=1.0)
+parser.add_argument(
+    "--tornasole_frequency", type=float, help="How often to save TS data", default=10
+)
+parser.add_argument("--run_name", type=str, help="Run Name", default=str(uuid.uuid4()))
+parser.add_argument("--local_reductions", nargs="+", type=str, default=[])
 # running in Tf estimator mode, script need to accept --model_dir parameter
-parser.add_argument('--model_dir', type=str, help="model dir", default=str(uuid.uuid4()) )
+parser.add_argument("--model_dir", type=str, help="model dir", default=str(uuid.uuid4()))
 args = parser.parse_args()
 # Network definition
-with tf.name_scope('foobar'):
+with tf.name_scope("foobar"):
     x = tf.placeholder(shape=(None, 2), dtype=tf.float32)
-    w = tf.Variable(initial_value=[[10.], [10.]])
-with tf.name_scope('foobaz'):
-    w0 = [[1], [1.]]
+    w = tf.Variable(initial_value=[[10.0], [10.0]])
+with tf.name_scope("foobaz"):
+    w0 = [[1], [1.0]]
     y = tf.matmul(x, w0)
 loss = tf.reduce_mean((tf.matmul(x, w) - y) ** 2, name="loss")
 global_step = tf.Variable(17, name="global_step", trainable=False)
-increment_global_step_op = tf.assign(global_step, global_step+1)
+increment_global_step_op = tf.assign(global_step, global_step + 1)
 optimizer = tf.train.AdamOptimizer(args.lr)
 optimizer = TornasoleOptimizer(optimizer)
 optimizer_op = optimizer.minimize(loss, global_step=increment_global_step_op)
 graph = tf.get_default_graph()
 list_of_tuples = [op.outputs for op in graph.get_operations()]
 t = str(time.time())
-hook = TornasoleHook("s3://tornasolecodebuildtest/container_testing/ts_outputs/tf"+t,
-		     save_config=SaveConfig(save_interval=10))
+hook = TornasoleHook(
+    "s3://tornasolecodebuildtest/container_testing/ts_outputs/tf" + t,
+    save_config=SaveConfig(save_interval=10),
+)
 sess = tf.train.MonitoredSession(hooks=[hook])
 for i in range(args.steps):
-  x_ = np.random.random((10, 2)) * args.scale
-  _loss, opt, gstep = sess.run([loss, optimizer_op, increment_global_step_op], {x: x_})
-  print (f'Step={i}, Loss={_loss}')
-
-
+    x_ = np.random.random((10, 2)) * args.scale
+    _loss, opt, gstep = sess.run([loss, optimizer_op, increment_global_step_op], {x: x_})
+    print(f"Step={i}, Loss={_loss}")
 
 
-#from tornasole.trials import create_trial
-#tr = create_trial('s3://tornasolecodebuildtest/container_testing/ts_outputs/tf'+t)
-#from tornasole.rules.generic import VanishingGradient
-#r = VanishingGradient(tr)
-#from tornasole.rules.rule_invoker import invoke_rule
-#invoke_rule(r, start_step=0, end_step=80)
+# from tornasole.trials import create_trial
+# tr = create_trial('s3://tornasolecodebuildtest/container_testing/ts_outputs/tf'+t)
+# from tornasole.rules.generic import VanishingGradient
+# r = VanishingGradient(tr)
+# from tornasole.rules.rule_invoker import invoke_rule
+# invoke_rule(r, start_step=0, end_step=80)