updated readme (added templating system), swapped demos to use built-in algorithm-python path; renamed serve() to init()

bafxyz · bafxyz · commit e4ac3cb2cb65 · 2021-03-03T12:52:50.000-04:00
diff --git a/.gitignore b/.gitignore
@@ -128,3 +128,4 @@ dmypy.json
 # Pyre type checker
 .pyre/
 .idea
+node_modules/
diff --git a/README.md b/README.md
@@ -1,21 +1,28 @@
 # Algorithm Development Kit (ADK), Python edition
 
+<!-- embedme examples/hello_world/src/Algorithm.py -->
 ```python
-import Algorithmia
-from adk import ADK
+from Algorithmia import ADK
 
 
 # API calls will begin at the apply() method, with the request body passed as 'input'
 # For more details, see algorithmia.com/developers/algorithm-development/languages
 
 def apply(input):
+    # If your apply function uses state that's loaded into memory via load, you can pass that loaded state to your apply
+    # function by defining an additional "state" parameter in your apply function; but it's optional!
     return "hello {}".format(str(input))
 
 
-algo = ADK(apply)
-algo.serve("Algorithmia")
-```
+# This turns your library code into an algorithm that can run on the platform.
+# If you intend to use loading operations, remember to pass a `load` function as a second variable.
+algorithm = ADK(apply)
+# The 'serve()' function actually starts the algorithm, you can follow along in the source code
+# to see how everything works.
+algorithm.init("Algorithmia")
 
+```
+## Focus
 This document will describe the following:
 - What is an Algorithm Development Kit
 - Changes to Algorithm development
@@ -37,22 +44,182 @@ This kit, when implemented by an algorithm developer - enables an easy way to ge
 
 Algorithm development does change with this introduction:
 - Primary development file has been renamed to `src/Algorithm.py` to aide in understanding around what this file actually does / why it's important
-- An additional import (`from adk import ADK`)
+- An additional import (`from algorithm import ADK`)
 - An optional `load()` function that can be implemented
     - This enables a dedicated function for preparing your algorithm for runtime operations, such as model loading, configuration, etc
 - A call to the handler function with your `apply` and optional` load` functions as inputs
     - ```python
-      algo = ADK(apply)
-      algo.serve("Algorithmia")
+      algorithm = ADK(apply)
+      algorithm.init("Algorithmia")
       ```
     - Converts the project into an executable, rather than a library
       - Which will interact with the `langserver` service on Algorithmia
       - But is debuggable via stdin/stdout when executed locally / outside of an Algorithm container
-        - When a payload is provided to `serve()`, that payload will be directly provided to your algorithm when executed locally, bypassing stdin parsing and simplifying debugging!
+        - When a payload is provided to `init()`, that payload will be directly provided to your algorithm when executed locally, bypassing stdin parsing and simplifying debugging!
       - This includes being able to step through your algorithm code in your IDE of choice! Just execute your `src/Algorithm.py` script and try stepping through your code with your favorite IDE
 
 ## Example workflows
 Check out these examples to help you get started:
-- [hello world example](examples/hello_world)
-- [hello world example with loaded state](examples/loaded_state_hello_world)
-- [pytorch based image classification](examples/pytorch_image_classification)
+### [hello world example](examples/hello_world)
+  <!-- embedme examples/hello_world/src/Algorithm.py -->
+```python
+from Algorithmia import ADK
+
+
+# API calls will begin at the apply() method, with the request body passed as 'input'
+# For more details, see algorithmia.com/developers/algorithm-development/languages
+
+def apply(input):
+    # If your apply function uses state that's loaded into memory via load, you can pass that loaded state to your apply
+    # function by defining an additional "state" parameter in your apply function; but it's optional!
+    return "hello {}".format(str(input))
+
+
+# This turns your library code into an algorithm that can run on the platform.
+# If you intend to use loading operations, remember to pass a `load` function as a second variable.
+algorithm = ADK(apply)
+# The 'serve()' function actually starts the algorithm, you can follow along in the source code
+# to see how everything works.
+algorithm.init("Algorithmia")
+
+```
+
+### [hello world example with loaded state](examples/loaded_state_hello_world)
+<!-- embedme examples/loaded_state_hello_world/src/Algorithm.py -->
+```python
+from Algorithmia import ADK
+
+
+# API calls will begin at the apply() method, with the request body passed as 'input'
+# For more details, see algorithmia.com/developers/algorithm-development/languages
+
+def apply(input, state):
+    # If your apply function uses state that's loaded into memory via load, you can pass that loaded state to your apply
+    # function by defining an additional "state" parameter in your apply function.
+    return "hello {} {}".format(str(input), str(state))
+
+
+def load():
+    # Here you can optionally define a function that will be called when the algorithm is loaded.
+    # The return object from this function can be passed directly as input to your apply function.
+    # A great example would be any model files that need to be available to this algorithm
+    # during runtime.
+    # Any variables returned here, will be passed as the secondary argument to your 'algorithm' function
+
+    return "Loading has been completed."
+
+
+# This turns your library code into an algorithm that can run on the platform.
+# If you intend to use loading operations, remember to pass a `load` function as a second variable.
+algorithm = ADK(apply, load)
+# The 'serve()' function actually starts the algorithm, you can follow along in the source code
+# to see how everything works.
+algorithm.init("Algorithmia")
+
+```
+## [pytorch based image classification](examples/pytorch_image_classification)
+<!-- embedme examples/pytorch_image_classification/src/Algorithm.py -->
+```python
+from Algorithmia import client, ADK
+import torch
+from PIL import Image
+import json
+from torchvision import models, transforms
+
+CLIENT = client()
+SMID_ALGO = "algo://util/SmartImageDownloader/0.2.x"
+LABEL_PATH = "data://AlgorithmiaSE/image_cassification_demo/imagenet_class_index.json"
+MODEL_PATHS = {
+    "squeezenet": 'data://AlgorithmiaSE/image_cassification_demo/squeezenet1_1-f364aa15.pth',
+    'alexnet': 'data://AlgorithmiaSE/image_cassification_demo/alexnet-owt-4df8aa71.pth',
+}
+
+
+def load_labels():
+    local_path = CLIENT.file(LABEL_PATH).getFile().name
+    with open(local_path) as f:
+        labels = json.load(f)
+    labels = [labels[str(k)][1] for k in range(len(labels))]
+    return labels
+
+
+def load_model(name):
+    if name == "squeezenet":
+        model = models.squeezenet1_1()
+        models.densenet121()
+        weights = torch.load(CLIENT.file(MODEL_PATHS['squeezenet']).getFile().name)
+    else:
+        model = models.alexnet()
+        weights = torch.load(CLIENT.file(MODEL_PATHS['alexnet']).getFile().name)
+    model.load_state_dict(weights)
+    return model.float().eval()
+
+
+def get_image(image_url):
+    input = {"image": image_url, "resize": {'width': 224, 'height': 224}}
+    result = CLIENT.algo(SMID_ALGO).pipe(input).result["savePath"][0]
+    local_path = CLIENT.file(result).getFile().name
+    img_data = Image.open(local_path)
+    return img_data
+
+
+def infer_image(image_url, n, state):
+    model = state['model']
+    labels = state['labels']
+    image_data = get_image(image_url)
+    transformed = transforms.Compose([
+        transforms.ToTensor(),
+        transforms.Normalize(mean=[0.485, 0.456, 0.406],
+                             std=[0.229, 0.224, 0.225])])
+    img_tensor = transformed(image_data).unsqueeze(dim=0)
+    infered = model.forward(img_tensor)
+    preds, indicies = torch.sort(torch.softmax(infered.squeeze(), dim=0), descending=True)
+    predicted_values = preds.detach().numpy()
+    indicies = indicies.detach().numpy()
+    result = []
+    for i in range(n):
+        label = labels[indicies[i]].lower().replace("_", " ")
+        confidence = float(predicted_values[i])
+        result.append({"label": label, "confidence": confidence})
+    return result
+
+
+def load():
+    output = {'model': load_model("squeezenet"), 'labels': load_labels()}
+    return output
+
+
+def apply(input, state):
+    if isinstance(input, dict):
+        if "n" in input:
+            n = input['n']
+        else:
+            n = 3
+        if "data" in input:
+            if isinstance(input['data'], str):
+                output = infer_image(input['data'], n, state)
+            elif isinstance(input['data'], list):
+                for row in input['data']:
+                    row['predictions'] = infer_image(row['image_url'], n, state)
+                output = input['data']
+            else:
+                raise Exception("'data' must be a image url or a list of image urls (with labels)")
+            return output
+        else:
+            raise Exception("'data' must be defined")
+    else:
+        raise Exception('input must be a json object')
+
+
+algorithm = ADK(apply_func=apply, load_func=load)
+algorithm.init({"data": "https://i.imgur.com/bXdORXl.jpeg"})
+
+```
+
+### Readme publishing
+To compile the template readme, please check out [embedme](https://github.com/zakhenry/embedme) utility
+and run the following:
+```commandline
+npm install -g npx
+npx embedme --stdout README_template.md > README.md
+```
diff --git a/README_template.md b/README_template.md
@@ -0,0 +1,64 @@
+# Algorithm Development Kit (ADK), Python edition
+
+<!-- embedme examples/hello_world/src/Algorithm.py -->
+```python
+```
+## Focus
+This document will describe the following:
+- What is an Algorithm Development Kit
+- Changes to Algorithm development
+- Example workflows you can use to create your own Algorithms.
+
+
+## What is an Algorithm Development Kit
+An Algorithm Development Kit is a package that contains all of the necessary components to convert a regular application into one that can be executed and run on Algorithmia.
+To do that, an ADK must be able to communicate with [langserver](https://github.com/algorithmiaio/langpacks/blob/develop/langpack_guide.md).
+To keep things simple, an ADK exposes some optional functions, along with an `apply` function that acts as the explicit entrypoint into your algorithm.
+Along with those basics, the ADK also exposes the ability to execute your algorithm locally, without `langserver`; which enables better debuggability.
+
+![adk architecture](assets/adk_architecture.png)
+
+This kit, when implemented by an algorithm developer - enables an easy way to get started with your project, along with well defined hooks to integrate with an existing project.
+
+
+## Changes to Algorithm Development
+
+Algorithm development does change with this introduction:
+- Primary development file has been renamed to `src/Algorithm.py` to aide in understanding around what this file actually does / why it's important
+- An additional import (`from algorithm import ADK`)
+- An optional `load()` function that can be implemented
+    - This enables a dedicated function for preparing your algorithm for runtime operations, such as model loading, configuration, etc
+- A call to the handler function with your `apply` and optional` load` functions as inputs
+    - ```python
+      algorithm = ADK(apply)
+      algorithm.init("Algorithmia")
+      ```
+    - Converts the project into an executable, rather than a library
+      - Which will interact with the `langserver` service on Algorithmia
+      - But is debuggable via stdin/stdout when executed locally / outside of an Algorithm container
+        - When a payload is provided to `init()`, that payload will be directly provided to your algorithm when executed locally, bypassing stdin parsing and simplifying debugging!
+      - This includes being able to step through your algorithm code in your IDE of choice! Just execute your `src/Algorithm.py` script and try stepping through your code with your favorite IDE
+
+## Example workflows
+Check out these examples to help you get started:
+### [hello world example](examples/hello_world)
+  <!-- embedme examples/hello_world/src/Algorithm.py -->
+```python
+```
+
+### [hello world example with loaded state](examples/loaded_state_hello_world)
+<!-- embedme examples/loaded_state_hello_world/src/Algorithm.py -->
+```python
+```
+## [pytorch based image classification](examples/pytorch_image_classification)
+<!-- embedme examples/pytorch_image_classification/src/Algorithm.py -->
+```python
+```
+
+### Readme publishing
+To compile the template readme, please check out [embedme](https://github.com/zakhenry/embedme) utility
+and run the following:
+```commandline
+npm install -g npx
+npx embedme --stdout README_template.md > README.md
+```
diff --git a/adk/ADK.py b/adk/ADK.py
@@ -125,7 +125,7 @@ def process_loop(self):
             finally:
                 self.write_to_pipe(response_obj)
 
-    def serve(self, local_payload=None):
+    def init(self, local_payload=None):
         try:
             if self.load_func:
                 self.load()
diff --git a/examples/hello_world/requirements.txt b/examples/hello_world/requirements.txt
@@ -1,2 +1 @@
-algorithmia>=1.7,<2
-algorithmia-adk>=1.0,<2
+algorithmia>=1.7,<2
diff --git a/examples/hello_world/src/Algorithm.py b/examples/hello_world/src/Algorithm.py
@@ -1,4 +1,4 @@
-from adk import ADK
+from Algorithmia import ADK
 
 
 # API calls will begin at the apply() method, with the request body passed as 'input'
@@ -12,7 +12,7 @@ def apply(input):
 
 # This turns your library code into an algorithm that can run on the platform.
 # If you intend to use loading operations, remember to pass a `load` function as a second variable.
-algo = ADK(apply)
+algorithm = ADK(apply)
 # The 'serve()' function actually starts the algorithm, you can follow along in the source code
 # to see how everything works.
-algo.serve("Algorithmia")
+algorithm.init("Algorithmia")
diff --git a/examples/loaded_state_hello_world/requirements.txt b/examples/loaded_state_hello_world/requirements.txt
@@ -1,2 +1 @@
-algorithmia>=1.7,<2
-algorithmia-adk>=1.0,<2
+algorithmia>=1.7,<2
diff --git a/examples/loaded_state_hello_world/src/Algorithm.py b/examples/loaded_state_hello_world/src/Algorithm.py
@@ -1,4 +1,4 @@
-from adk import ADK
+from Algorithmia import ADK
 
 
 # API calls will begin at the apply() method, with the request body passed as 'input'
@@ -22,7 +22,7 @@ def load():
 
 # This turns your library code into an algorithm that can run on the platform.
 # If you intend to use loading operations, remember to pass a `load` function as a second variable.
-algo = ADK(apply, load)
+algorithm = ADK(apply, load)
 # The 'serve()' function actually starts the algorithm, you can follow along in the source code
 # to see how everything works.
-algo.serve("Algorithmia")
+algorithm.init("Algorithmia")
diff --git a/examples/pytorch_image_classification/requirements.txt b/examples/pytorch_image_classification/requirements.txt
@@ -1,5 +1,4 @@
 algorithmia>=1.7,<2
-algorithmia-adk
 six
 Pillow==8.1.0
 torch==1.6.0
diff --git a/examples/pytorch_image_classification/src/Algorithm.py b/examples/pytorch_image_classification/src/Algorithm.py
@@ -1,11 +1,10 @@
-import Algorithmia
-from adk import ADK
+from Algorithmia import client, ADK
 import torch
 from PIL import Image
 import json
 from torchvision import models, transforms
 
-CLIENT = Algorithmia.client()
+CLIENT = client()
 SMID_ALGO = "algo://util/SmartImageDownloader/0.2.x"
 LABEL_PATH = "data://AlgorithmiaSE/image_cassification_demo/imagenet_class_index.json"
 MODEL_PATHS = {
@@ -90,5 +89,5 @@ def apply(input, state):
         raise Exception('input must be a json object')
 
 
-algo = ADK(apply_func=apply, load_func=load)
-algo.serve({"data": "https://i.imgur.com/bXdORXl.jpeg"})
+algorithm = ADK(apply_func=apply, load_func=load)
+algorithm.init({"data": "https://i.imgur.com/bXdORXl.jpeg"})
diff --git a/setup.py b/setup.py
@@ -6,7 +6,7 @@
     name='algorithmia-adk',
     version='0.1.0',
     description='adk python ADK client',
-    long_description='adk Development Kit code used for creating Python algorithms on adk.',
+    long_description='adk Development Kit code used for creating Python algorithms on adk. Built into the Algorithmia client',
     url='http://github.com/algorithmiaio/algorithmia-adk-python',
     license='MIT',
     author='Algorithmia',

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1 @@`
`1`		`-algorithmia>=1.7,<2`
`2`		`-algorithmia-adk>=1.0,<2`
	`1`	`+algorithmia>=1.7,<2`
Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,4 @@`
`1`	`1`	`algorithmia>=1.7,<2`
`2`		`-algorithmia-adk`
`3`	`2`	`six`
`4`	`3`	`Pillow==8.1.0`
`5`	`4`	`torch==1.6.0`