|
5 | 5 | "id": "2c31f83d-be31-47b9-af3b-a4285576b4db", |
6 | 6 | "metadata": {}, |
7 | 7 | "source": [ |
8 | | - "Host Llama2-70B on Amazon SageMaker using LMI V7 container" |
| 8 | + "# Host Llama2-70B on Amazon SageMaker using TRT-LLM LMI container" |
| 9 | + ] |
| 10 | + }, |
| 11 | + { |
| 12 | + "cell_type": "markdown", |
| 13 | + "id": "5d63a9c2", |
| 14 | + "metadata": {}, |
| 15 | + "source": [ |
| 16 | + "---\n", |
| 17 | + "\n", |
| 18 | + "This notebooks CI test result for us-west-2 is as follows. CI test results in other regions can be found at the end of the notebook.\n", |
| 19 | + "\n", |
| 20 | + "\n", |
| 21 | + "\n", |
| 22 | + "---" |
9 | 23 | ] |
10 | 24 | }, |
11 | 25 | { |
|
35 | 49 | }, |
36 | 50 | "outputs": [], |
37 | 51 | "source": [ |
38 | | - "%pip install sagemaker boto3 --upgrade" |
| 52 | + "%pip install sagemaker boto3 awscli huggingface_hub --upgrade --quiet" |
39 | 53 | ] |
40 | 54 | }, |
41 | 55 | { |
|
79 | 93 | "sess = sagemaker.session.Session() # sagemaker session for interacting with different AWS APIs\n", |
80 | 94 | "bucket = sess.default_bucket() # bucket to house artifacts\n", |
81 | 95 | "model_bucket = sess.default_bucket() # bucket to house artifacts\n", |
82 | | - "s3_code_prefix = \"hf-large-model-djl/code_llama2\" # folder within bucket where code artifact will go\n", |
83 | | - "\n", |
| 96 | + "s3_code_prefix = \"hf-large-model-djl/code_llama2\"\n", |
84 | 97 | "region = sess._region_name\n", |
85 | 98 | "account_id = sess.account_id()\n", |
86 | 99 | "\n", |
|
169 | 182 | "tags": [] |
170 | 183 | }, |
171 | 184 | "source": [ |
172 | | - "## Create serving.properties file, upload model to S3 and provid ethe inference container\n", |
| 185 | + "## Create serving.properties file, upload model to S3 and provide the inference container\n", |
173 | 186 | "\n", |
174 | 187 | "SageMaker Large Model Inference containers can be used to host models without any additional inference code. You also have the option to provide your inference script if you need any custom pre-processing of the input data or postprocessing of the model's predictions.\n", |
175 | 188 | "\n", |
|
202 | 215 | "- `engine`: The engine for DJL to use. \n", |
203 | 216 | "- `option.model_id`: This can be the S3 uri of the pre-trained model or the model id of a pretrained model hosted inside a model repository on huggingface.co (https://huggingface.co/models).\n", |
204 | 217 | "- `option.tensor_parallel_degree`: Set to the number of GPU devices over which Accelerate needs to partition the model. This parameter also controls the no of workers per model which will be started up when DJL serving runs. As an example if we have a 8 GPU machine and we are creating 8 partitions then we will have 1 worker per model to serve the requests.\n", |
205 | | - "- `option.rolling_batch` : This parameter enables rolling batch of inputs\n", |
206 | 218 | "- `option.max_rolling_batch_size`: Sets the max batch size \n", |
207 | 219 | "- `option.model_loading_timeout` : Sets the timeout value for downloading and loading the model to serve inference\n", |
208 | 220 | "\n", |
|
227 | 239 | "outputs": [], |
228 | 240 | "source": [ |
229 | 241 | "%%writefile ./code_llama2/serving.properties\n", |
230 | | - "engine=MPI\n", |
231 | | - "option.model_id={{s3url}}\n", |
232 | | - "option.tensor_parallel_degree=8\n", |
233 | | - "option.use_custom_all_reduce=true\n", |
234 | | - "option.output_formatter=json\n", |
235 | | - "option.max_rolling_batch_size=64\n", |
236 | | - "option.model_loading_timeout=3600" |
| 242 | + "engine = MPI\n", |
| 243 | + "option.model_id = {{s3url}}\n", |
| 244 | + "option.tensor_parallel_degree = 8\n", |
| 245 | + "option.use_custom_all_reduce = true\n", |
| 246 | + "option.output_formatter = json\n", |
| 247 | + "option.max_rolling_batch_size = 64\n", |
| 248 | + "option.model_loading_timeout = 3600" |
237 | 249 | ] |
238 | 250 | }, |
239 | 251 | { |
|
270 | 282 | }, |
271 | 283 | "outputs": [], |
272 | 284 | "source": [ |
273 | | - "\n", |
274 | 285 | "inference_image_uri = sagemaker.image_uris.retrieve(\n", |
275 | 286 | " framework=\"djl-tensorrtllm\", region=region, version=\"0.25.0\"\n", |
276 | 287 | ")\n", |
277 | | - "print(f\"Image going to be used is ---- > {inference_image_uri}\")\n" |
| 288 | + "print(f\"Image going to be used is ---- > {inference_image_uri}\")" |
278 | 289 | ] |
279 | 290 | }, |
280 | 291 | { |
|
360 | 371 | "create_model_response = sm_client.create_model(\n", |
361 | 372 | " ModelName=model_name,\n", |
362 | 373 | " ExecutionRoleArn=role,\n", |
363 | | - " PrimaryContainer={\"Image\": inference_image_uri,\n", |
364 | | - " \"ModelDataUrl\": s3_code_artifact,\n", |
365 | | - " 'Environment': {'MODEL_LOADING_TIMEOUT': '3600'}\n", |
366 | | - " }\n", |
| 374 | + " PrimaryContainer={\n", |
| 375 | + " \"Image\": inference_image_uri,\n", |
| 376 | + " \"ModelDataUrl\": s3_code_artifact,\n", |
| 377 | + " \"Environment\": {\"MODEL_LOADING_TIMEOUT\": \"3600\"},\n", |
| 378 | + " },\n", |
367 | 379 | ")\n", |
368 | 380 | "model_arn = create_model_response[\"ModelArn\"]\n", |
369 | 381 | "\n", |
|
475 | 487 | " \"parameters\": {\n", |
476 | 488 | " \"do_sample\": True,\n", |
477 | 489 | " \"max_new_tokens\": 256,\n", |
478 | | - " \"top_k\" : 5,\n", |
| 490 | + " \"top_k\": 5,\n", |
479 | 491 | " },\n", |
480 | 492 | " }\n", |
481 | 493 | " ),\n", |
|
530 | 542 | ] |
531 | 543 | }, |
532 | 544 | { |
533 | | - "cell_type": "code", |
534 | | - "execution_count": null, |
535 | | - "id": "b0c4f2f8-0474-473b-9e75-ff55ddcfffdf", |
| 545 | + "cell_type": "markdown", |
| 546 | + "id": "bc98aaf3", |
536 | 547 | "metadata": {}, |
537 | | - "outputs": [], |
538 | | - "source": [] |
| 548 | + "source": [ |
| 549 | + "## Notebook CI Test Results\n", |
| 550 | + "\n", |
| 551 | + "This notebook was tested in multiple regions. The test results are as follows, except for us-west-2 which is shown at the top of the notebook.\n", |
| 552 | + "\n", |
| 553 | + "\n", |
| 554 | + "\n", |
| 555 | + "\n", |
| 556 | + "\n", |
| 557 | + "\n", |
| 558 | + "\n", |
| 559 | + "\n", |
| 560 | + "\n", |
| 561 | + "\n", |
| 562 | + "\n", |
| 563 | + "\n", |
| 564 | + "\n", |
| 565 | + "\n", |
| 566 | + "\n", |
| 567 | + "\n", |
| 568 | + "\n", |
| 569 | + "\n", |
| 570 | + "\n", |
| 571 | + "\n", |
| 572 | + "\n", |
| 573 | + "\n", |
| 574 | + "\n", |
| 575 | + "\n", |
| 576 | + "\n", |
| 577 | + "\n", |
| 578 | + "\n", |
| 579 | + "\n", |
| 580 | + "\n", |
| 581 | + "\n" |
| 582 | + ] |
539 | 583 | } |
540 | 584 | ], |
541 | 585 | "metadata": { |
|
0 commit comments