Skip to content

Commit 19c3c3f

Browse files
ZhankuilZhankuil
committed
documentation: improvements to SageMaker pipeline documentation
* fix errors in SageMaker pipelines docstrings * add code snippets
1 parent efdf3ec commit 19c3c3f

File tree

10 files changed

+144
-71
lines changed

10 files changed

+144
-71
lines changed

‎doc/workflows/pipelines/index.rst

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
################
2-
Amazon Pipelines
3-
################
1+
###################
2+
SageMaker Pipelines
3+
###################
44

5-
SageMaker APIs for creating and managing Amazon Pipelines.
5+
SageMaker APIs for creating and managing SageMaker Pipelines.
66

77
.. toctree::
88
:maxdepth: 2

‎doc/workflows/pipelines/sagemaker.workflow.pipelines.rst

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,6 @@ Conditions
3333

3434
.. autoclass:: sagemaker.workflow.conditions.ConditionOr
3535

36-
.. autofunction:: sagemaker.workflow.conditions.primitive_or_expr
37-
3836
Entities
3937
--------
4038

@@ -44,7 +42,7 @@ Entities
4442

4543
.. autoclass:: sagemaker.workflow.entities.Expression
4644

47-
Execution_variables
45+
Execution Variables
4846
-------------------
4947

5048
.. autoclass:: sagemaker.workflow.execution_variables.ExecutionVariable
@@ -73,12 +71,17 @@ Pipeline
7371
--------
7472

7573
.. autoclass:: sagemaker.workflow.pipeline.Pipeline
74+
:members:
75+
76+
.. autoclass:: sagemaker.workflow.pipeline._PipelineExecution
77+
:members:
7678

77-
.. autofunction:: sagemaker.workflow.pipeline.format_start_parameters
79+
Pipeline Experiment Config
80+
--------------------------
7881

79-
.. autofunction:: sagemaker.workflow.pipeline.interpolate
82+
.. autoclass:: sagemaker.workflow.pipeline_experiment_config.PipelineExperimentConfig
8083

81-
.. autofunction:: sagemaker.workflow.pipeline.update_args
84+
.. autoclass:: sagemaker.workflow.pipeline_experiment_config.PipelineExperimentConfigProperty
8285

8386
Properties
8487
----------
@@ -115,7 +118,4 @@ Steps
115118

116119
.. autoclass:: sagemaker.workflow.steps.ProcessingStep
117120

118-
Utilities
119-
---------
120-
121-
.. autofunction:: sagemaker.workflow.utilities.list_to_request
121+
.. autoclass:: sagemaker.workflow.steps.CacheConfig

‎src/sagemaker/workflow/condition_step.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,10 +55,10 @@ def __init__(
5555
conditions (List[Condition]): A list of `sagemaker.workflow.conditions.Condition`
5656
instances.
5757
if_steps (List[Union[Step, StepCollection]]): A list of `sagemaker.workflow.steps.Step`
58-
and `sagemaker.workflow.step_collections.StepCollection` instances that are
58+
or `sagemaker.workflow.step_collections.StepCollection` instances that are
5959
marked as ready for execution if the list of conditions evaluates to True.
6060
else_steps (List[Union[Step, StepCollection]]): A list of `sagemaker.workflow.steps.Step`
61-
and `sagemaker.workflow.step_collections.StepCollection` instances that are
61+
or `sagemaker.workflow.step_collections.StepCollection` instances that are
6262
marked as ready for execution if the list of conditions evaluates to False.
6363
"""
6464
super(ConditionStep, self).__init__(name, StepTypeEnum.CONDITION, depends_on)

‎src/sagemaker/workflow/conditions.py

Lines changed: 43 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -66,13 +66,13 @@ class ConditionComparison(Condition):
6666
"""Generic comparison condition that can be used to derive specific condition comparisons.
6767
6868
Attributes:
69-
left (ConditionValueType): The execution variable, parameter, or
70-
property to use in the comparison.
69+
left (Union[ConditionValueType, PrimitiveType]): The execution variable, parameter,
70+
property, or Python primitive value to use in the comparison.
7171
right (Union[ConditionValueType, PrimitiveType]): The execution variable,
7272
parameter, property, or Python primitive value to compare to.
7373
"""
7474

75-
left: ConditionValueType = attr.ib(default=None)
75+
left: Union[ConditionValueType, PrimitiveType] = attr.ib(default=None)
7676
right: Union[ConditionValueType, PrimitiveType] = attr.ib(default=None)
7777

7878
def to_request(self) -> RequestType:
@@ -87,12 +87,16 @@ def to_request(self) -> RequestType:
8787
class ConditionEquals(ConditionComparison):
8888
"""A condition for equality comparisons."""
8989

90-
def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, PrimitiveType]):
90+
def __init__(
91+
self,
92+
left: Union[ConditionValueType, PrimitiveType],
93+
right: Union[ConditionValueType, PrimitiveType],
94+
):
9195
"""Construct A condition for equality comparisons.
9296
9397
Args:
94-
left (ConditionValueType): The execution variable, parameter,
95-
or property to use in the comparison.
98+
left (Union[ConditionValueType, PrimitiveType]): The execution variable,
99+
parameter, property, or Python primitive value to use in the comparison.
96100
right (Union[ConditionValueType, PrimitiveType]): The execution
97101
variable, parameter, property, or Python primitive value to compare to.
98102
"""
@@ -103,12 +107,16 @@ def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, Pr
103107
class ConditionGreaterThan(ConditionComparison):
104108
"""A condition for greater than comparisons."""
105109

106-
def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, PrimitiveType]):
110+
def __init__(
111+
self,
112+
left: Union[ConditionValueType, PrimitiveType],
113+
right: Union[ConditionValueType, PrimitiveType],
114+
):
107115
"""Construct an instance of ConditionGreaterThan for greater than comparisons.
108116
109117
Args:
110-
left (ConditionValueType): The execution variable, parameter,
111-
or property to use in the comparison.
118+
left (Union[ConditionValueType, PrimitiveType]): The execution variable,
119+
parameter, property, or Python primitive value to use in the comparison.
112120
right (Union[ConditionValueType, PrimitiveType]): The execution
113121
variable, parameter, property, or Python primitive value to compare to.
114122
"""
@@ -119,12 +127,16 @@ def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, Pr
119127
class ConditionGreaterThanOrEqualTo(ConditionComparison):
120128
"""A condition for greater than or equal to comparisons."""
121129

122-
def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, PrimitiveType]):
130+
def __init__(
131+
self,
132+
left: Union[ConditionValueType, PrimitiveType],
133+
right: Union[ConditionValueType, PrimitiveType],
134+
):
123135
"""Construct of ConditionGreaterThanOrEqualTo for greater than or equal to comparisons.
124136
125137
Args:
126-
left (ConditionValueType): The execution variable, parameter,
127-
or property to use in the comparison.
138+
left (Union[ConditionValueType, PrimitiveType]): The execution variable,
139+
parameter, property, or Python primitive value to use in the comparison.
128140
right (Union[ConditionValueType, PrimitiveType]): The execution
129141
variable, parameter, property, or Python primitive value to compare to.
130142
"""
@@ -135,12 +147,16 @@ def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, Pr
135147
class ConditionLessThan(ConditionComparison):
136148
"""A condition for less than comparisons."""
137149

138-
def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, PrimitiveType]):
150+
def __init__(
151+
self,
152+
left: Union[ConditionValueType, PrimitiveType],
153+
right: Union[ConditionValueType, PrimitiveType],
154+
):
139155
"""Construct an instance of ConditionLessThan for less than comparisons.
140156
141157
Args:
142-
left (ConditionValueType): The execution variable, parameter,
143-
or property to use in the comparison.
158+
left (Union[ConditionValueType, PrimitiveType]): The execution variable,
159+
parameter, property, or Python primitive value to use in the comparison.
144160
right (Union[ConditionValueType, PrimitiveType]): The execution
145161
variable, parameter, property, or Python primitive value to compare to.
146162
"""
@@ -151,12 +167,16 @@ def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, Pr
151167
class ConditionLessThanOrEqualTo(ConditionComparison):
152168
"""A condition for less than or equal to comparisons."""
153169

154-
def __init__(self, left: ConditionValueType, right: Union[ConditionValueType, PrimitiveType]):
170+
def __init__(
171+
self,
172+
left: Union[ConditionValueType, PrimitiveType],
173+
right: Union[ConditionValueType, PrimitiveType],
174+
):
155175
"""Construct ConditionLessThanOrEqualTo for less than or equal to comparisons.
156176
157177
Args:
158-
left (ConditionValueType): The execution variable, parameter,
159-
or property to use in the comparison.
178+
left (Union[ConditionValueType, PrimitiveType]): The execution variable,
179+
parameter, property, or Python primitive value to use in the comparison.
160180
right (Union[ConditionValueType, PrimitiveType]): The execution
161181
variable, parameter, property, or Python primitive value to compare to.
162182
"""
@@ -168,13 +188,15 @@ class ConditionIn(Condition):
168188
"""A condition to check membership."""
169189

170190
def __init__(
171-
self, value: ConditionValueType, in_values: List[Union[ConditionValueType, PrimitiveType]]
191+
self,
192+
value: Union[ConditionValueType, PrimitiveType],
193+
in_values: List[Union[ConditionValueType, PrimitiveType]],
172194
):
173195
"""Construct a `ConditionIn` condition to check membership.
174196
175197
Args:
176-
value (ConditionValueType): The execution variable,
177-
parameter, or property to use for the in comparison.
198+
value (Union[ConditionValueType, PrimitiveType]): The execution variable,
199+
parameter, property or primitive value to check for membership.
178200
in_values (List[Union[ConditionValueType, PrimitiveType]]): The list
179201
of values to check for membership in.
180202
"""

‎src/sagemaker/workflow/execution_variables.py

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,7 @@ def expr(self) -> RequestType:
3737

3838

3939
class ExecutionVariables:
40-
"""Enum-like class for all ExecutionVariable instances.
41-
42-
Considerations to move these as module-level constants should be made.
43-
"""
40+
"""All available ExecutionVariable."""
4441

4542
START_DATETIME = ExecutionVariable("StartDateTime")
4643
CURRENT_DATETIME = ExecutionVariable("CurrentDateTime")

‎src/sagemaker/workflow/functions.py

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,19 @@
2424
class Join(Expression):
2525
"""Join together properties.
2626
27+
Examples:
28+
Build a Amazon S3 Uri with bucket name parameter and pipeline execution Id and use it
29+
as training input::
30+
31+
s3_bucket = ParameterString('bucket', default_value='my-bucket')
32+
33+
TrainingInput(
34+
s3_data=Join(on='/', ['s3:/', bucket, ExecutionVariables.PIPELINE_EXECUTION_ID]),
35+
content_type="text/csv")
36+
2737
Attributes:
28-
values (List[Union[PrimitiveType, Parameter]]): The primitive types
29-
and parameters to join.
38+
values (List[Union[PrimitiveType, Parameter, Expression]]):
39+
The primitive type values, parameters, step properties, expressions to join.
3040
on_str (str): The string to join the values on (Defaults to "").
3141
"""
3242

‎src/sagemaker/workflow/parameters.py

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ class Parameter(Entity):
5454
Attributes:
5555
name (str): The name of the parameter.
5656
parameter_type (ParameterTypeEnum): The type of the parameter.
57-
default_value (PrimitiveType): The default Python value of the parameter.
57+
default_value (PrimitiveType): The default value of the parameter.
5858
"""
5959

6060
name: str = attr.ib(factory=str)
@@ -143,7 +143,7 @@ def _check_default_value_type(cls, value, python_type):
143143

144144

145145
class ParameterString(Parameter, str):
146-
"""Pipeline string parameter for workflow."""
146+
"""String parameter for pipelines."""
147147

148148
def __new__(cls, *args, **kwargs): # pylint: disable=unused-argument
149149
"""Subclass str"""
@@ -155,7 +155,11 @@ def __init__(self, name: str, default_value: str = None, enum_values: List[str]
155155
156156
Args:
157157
name (str): The name of the parameter.
158-
default_value (str): The default Python value of the parameter. Defaults to None.
158+
default_value (str): The default value of the parameter.
159+
The default value could be overridden at start of an execution.
160+
If not set or it is set to None, a value must be provided
161+
at the start of the execution.
162+
enum_values (List[str]): Enum values for this parameter.
159163
"""
160164
super(ParameterString, self).__init__(
161165
name=name, parameter_type=ParameterTypeEnum.STRING, default_value=default_value
@@ -175,7 +179,7 @@ def to_request(self) -> RequestType:
175179

176180

177181
class ParameterInteger(Parameter, int):
178-
"""Pipeline string parameter for workflow."""
182+
"""Integer parameter for pipelines."""
179183

180184
def __new__(cls, *args, **kwargs):
181185
"""Subclass int"""
@@ -187,15 +191,18 @@ def __init__(self, name: str, default_value: int = None):
187191
188192
Args:
189193
name (str): The name of the parameter.
190-
default_value (int): The default Python value of the parameter.
194+
default_value (int): The default value of the parameter.
195+
The default value could be overridden at start of an execution.
196+
If not set or it is set to None, a value must be provided
197+
at the start of the execution.
191198
"""
192199
super(ParameterInteger, self).__init__(
193200
name=name, parameter_type=ParameterTypeEnum.INTEGER, default_value=default_value
194201
)
195202

196203

197204
class ParameterFloat(Parameter, float):
198-
"""Pipeline float parameter for workflow."""
205+
"""Float parameter for pipelines."""
199206

200207
def __new__(cls, *args, **kwargs):
201208
"""Subclass float"""
@@ -207,7 +214,10 @@ def __init__(self, name: str, default_value: float = None):
207214
208215
Args:
209216
name (str): The name of the parameter.
210-
default_value (float): The default Python value of the parameter.
217+
default_value (float): The default value of the parameter.
218+
The default value could be overridden at start of an execution.
219+
If not set or it is set to None, a value must be provided
220+
at the start of the execution.
211221
"""
212222
super(ParameterFloat, self).__init__(
213223
name=name, parameter_type=ParameterTypeEnum.FLOAT, default_value=default_value

‎src/sagemaker/workflow/pipeline.py

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -45,15 +45,15 @@ class Pipeline(Entity):
4545
4646
Attributes:
4747
name (str): The name of the pipeline.
48-
parameters (Sequence[Parameters]): The list of the parameters.
48+
parameters (Sequence[Parameter]): The list of the parameters.
4949
pipeline_experiment_config (Optional[PipelineExperimentConfig]): If set,
5050
the workflow will attempt to create an experiment and trial before
5151
executing the steps. Creation will be skipped if an experiment or a trial with
5252
the same name already exists. By default, pipeline name is used as
5353
experiment name and execution id is used as the trial name.
5454
If set to None, no experiment or trial will be created automatically.
55-
steps (Sequence[Steps]): The list of the non-conditional steps associated with the pipeline.
56-
Any steps that are within the
55+
steps (Sequence[Union[Step, StepCollection]]): The list of the non-conditional steps
56+
associated with the pipeline. Any steps that are within the
5757
`if_steps` or `else_steps` of a `ConditionStep` cannot be listed in the steps of a
5858
pipeline. Of particular note, the workflow service rejects any pipeline definitions that
5959
specify a step in the list of steps of a pipeline and that step in the `if_steps` or
@@ -139,7 +139,9 @@ def describe(self) -> Dict[str, Any]:
139139
"""Describes a Pipeline in the Workflow service.
140140
141141
Returns:
142-
Response dict from the service.
142+
Response dict from the service. See `boto3 client documentation
143+
<https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/\
144+
sagemaker.html#SageMaker.Client.describe_pipeline>`_
143145
"""
144146
return self.sagemaker_session.sagemaker_client.describe_pipeline(PipelineName=self.name)
145147

@@ -209,15 +211,15 @@ def delete(self) -> Dict[str, Any]:
209211

210212
def start(
211213
self,
212-
parameters: Dict[str, Any] = None,
214+
parameters: Dict[str, Union[str, bool, int, float]] = None,
213215
execution_display_name: str = None,
214216
execution_description: str = None,
215217
):
216218
"""Starts a Pipeline execution in the Workflow service.
217219
218220
Args:
219-
parameters (List[Dict[str, str]]): A list of parameter dicts of the form
220-
{"Name": "string", "Value": "string"}.
221+
parameters (Dict[str, Union[str, bool, int, float]]): values to override
222+
pipeline parameters.
221223
execution_display_name (str): The display name of the pipeline execution.
222224
execution_description (str): A description of the execution.
223225
@@ -370,13 +372,27 @@ def stop(self):
370372
)
371373

372374
def describe(self):
373-
"""Describes a pipeline execution."""
375+
"""Describes a pipeline execution.
376+
377+
Returns:
378+
Information about the pipeline execution. See
379+
`boto3 client describe_pipeline_execution
380+
<https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/\
381+
sagemaker.html#SageMaker.Client.describe_pipeline_execution>`_.
382+
"""
374383
return self.sagemaker_session.sagemaker_client.describe_pipeline_execution(
375384
PipelineExecutionArn=self.arn
376385
)
377386

378387
def list_steps(self):
379-
"""Describes a pipeline execution's steps."""
388+
"""Describes a pipeline execution's steps.
389+
390+
Returns:
391+
Information about the steps of the pipeline execution. See
392+
`boto3 client list_pipeline_execution_steps
393+
<https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/\
394+
sagemaker.html#SageMaker.Client.list_pipeline_execution_steps>`_.
395+
"""
380396
response = self.sagemaker_session.sagemaker_client.list_pipeline_execution_steps(
381397
PipelineExecutionArn=self.arn
382398
)

0 commit comments

Comments
 (0)